2024-11-26 20:40:25,940 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-26 20:40:25,955 main DEBUG Took 0.012822 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-26 20:40:25,955 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-26 20:40:25,956 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-26 20:40:25,957 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-26 20:40:25,959 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:40:25,967 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-26 20:40:25,984 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:40:25,986 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:40:25,987 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:40:25,988 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:40:25,988 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:40:25,989 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:40:25,990 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:40:25,990 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:40:25,991 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:40:25,992 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:40:25,993 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:40:25,993 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:40:25,994 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:40:25,994 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:40:25,995 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:40:25,995 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:40:25,996 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:40:25,996 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:40:26,004 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:40:26,005 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:40:26,009 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:40:26,009 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:40:26,010 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:40:26,010 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:40:26,011 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:40:26,011 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-26 20:40:26,013 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:40:26,015 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-26 20:40:26,017 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-26 20:40:26,018 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-26 20:40:26,019 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-26 20:40:26,020 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-26 20:40:26,031 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-26 20:40:26,034 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-26 20:40:26,036 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-26 20:40:26,037 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-26 20:40:26,037 main DEBUG createAppenders(={Console}) 2024-11-26 20:40:26,039 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-26 20:40:26,039 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-26 20:40:26,039 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-26 20:40:26,040 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-26 20:40:26,041 main DEBUG OutputStream closed 2024-11-26 20:40:26,041 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-26 20:40:26,041 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-26 20:40:26,042 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-26 20:40:26,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-26 20:40:26,149 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-26 20:40:26,150 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-26 20:40:26,152 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-26 20:40:26,153 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-26 20:40:26,153 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-26 20:40:26,153 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-26 20:40:26,154 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-26 20:40:26,155 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-26 20:40:26,155 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-26 20:40:26,156 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-26 20:40:26,156 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-26 20:40:26,158 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-26 20:40:26,159 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-26 20:40:26,159 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-26 20:40:26,160 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-26 20:40:26,160 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-26 20:40:26,161 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-26 20:40:26,163 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26 20:40:26,164 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-26 20:40:26,164 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-26 20:40:26,165 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-26T20:40:26,524 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49 2024-11-26 20:40:26,530 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-26 20:40:26,530 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26T20:40:26,544 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-26T20:40:26,592 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-26T20:40:26,613 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/cluster_bc121de7-746a-3d7e-5c7c-57aa2e103a17, deleteOnExit=true 2024-11-26T20:40:26,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-26T20:40:26,615 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/test.cache.data in system properties and HBase conf 2024-11-26T20:40:26,616 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T20:40:26,617 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/hadoop.log.dir in system properties and HBase conf 2024-11-26T20:40:26,617 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T20:40:26,618 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T20:40:26,618 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T20:40:26,755 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-26T20:40:26,891 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:40:26,896 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T20:40:26,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T20:40:26,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T20:40:26,898 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T20:40:26,899 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T20:40:26,899 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T20:40:26,900 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T20:40:26,900 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T20:40:26,901 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T20:40:26,902 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/nfs.dump.dir in system properties and HBase conf 2024-11-26T20:40:26,902 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/java.io.tmpdir in system properties and HBase conf 2024-11-26T20:40:26,903 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T20:40:26,903 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T20:40:26,904 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T20:40:27,905 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-26T20:40:28,005 INFO [Time-limited test {}] log.Log(170): Logging initialized @2980ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-26T20:40:28,109 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T20:40:28,204 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T20:40:28,238 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T20:40:28,238 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T20:40:28,240 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T20:40:28,257 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T20:40:28,260 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/hadoop.log.dir/,AVAILABLE} 2024-11-26T20:40:28,261 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T20:40:28,523 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/java.io.tmpdir/jetty-localhost-34349-hadoop-hdfs-3_4_1-tests_jar-_-any-559463712830371141/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T20:40:28,531 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:34349} 2024-11-26T20:40:28,531 INFO [Time-limited test {}] server.Server(415): Started @3507ms 2024-11-26T20:40:28,979 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T20:40:28,987 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T20:40:28,988 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T20:40:28,988 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T20:40:28,988 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T20:40:28,989 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/hadoop.log.dir/,AVAILABLE} 2024-11-26T20:40:28,990 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T20:40:29,116 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/java.io.tmpdir/jetty-localhost-35529-hadoop-hdfs-3_4_1-tests_jar-_-any-12398676713571276907/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T20:40:29,116 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:35529} 2024-11-26T20:40:29,117 INFO [Time-limited test {}] server.Server(415): Started @4092ms 2024-11-26T20:40:29,175 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T20:40:29,760 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/cluster_bc121de7-746a-3d7e-5c7c-57aa2e103a17/data/data2/current/BP-376524052-172.17.0.2-1732653627608/current, will proceed with Du for space computation calculation, 2024-11-26T20:40:29,760 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/cluster_bc121de7-746a-3d7e-5c7c-57aa2e103a17/data/data1/current/BP-376524052-172.17.0.2-1732653627608/current, will proceed with Du for space computation calculation, 2024-11-26T20:40:29,825 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T20:40:29,900 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5f7ca23372f44bf4 with lease ID 0x405ffb8686fa8674: Processing first storage report for DS-a4c67829-1b02-431d-831c-8acb2010147f from datanode DatanodeRegistration(127.0.0.1:34989, datanodeUuid=6f31a597-6d37-4855-9b75-9219c1156dc9, infoPort=38509, infoSecurePort=0, ipcPort=41175, storageInfo=lv=-57;cid=testClusterID;nsid=1814991287;c=1732653627608) 2024-11-26T20:40:29,903 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5f7ca23372f44bf4 with lease ID 0x405ffb8686fa8674: from storage DS-a4c67829-1b02-431d-831c-8acb2010147f node DatanodeRegistration(127.0.0.1:34989, datanodeUuid=6f31a597-6d37-4855-9b75-9219c1156dc9, infoPort=38509, infoSecurePort=0, ipcPort=41175, storageInfo=lv=-57;cid=testClusterID;nsid=1814991287;c=1732653627608), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-26T20:40:29,904 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5f7ca23372f44bf4 with lease ID 0x405ffb8686fa8674: Processing first storage report for DS-19de84de-d5ec-4166-8f5b-2f7affef8a3f from datanode DatanodeRegistration(127.0.0.1:34989, datanodeUuid=6f31a597-6d37-4855-9b75-9219c1156dc9, infoPort=38509, infoSecurePort=0, ipcPort=41175, storageInfo=lv=-57;cid=testClusterID;nsid=1814991287;c=1732653627608) 2024-11-26T20:40:29,904 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5f7ca23372f44bf4 with lease ID 0x405ffb8686fa8674: from storage DS-19de84de-d5ec-4166-8f5b-2f7affef8a3f node DatanodeRegistration(127.0.0.1:34989, datanodeUuid=6f31a597-6d37-4855-9b75-9219c1156dc9, infoPort=38509, infoSecurePort=0, ipcPort=41175, storageInfo=lv=-57;cid=testClusterID;nsid=1814991287;c=1732653627608), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T20:40:29,972 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49 2024-11-26T20:40:30,084 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/cluster_bc121de7-746a-3d7e-5c7c-57aa2e103a17/zookeeper_0, clientPort=50761, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/cluster_bc121de7-746a-3d7e-5c7c-57aa2e103a17/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/cluster_bc121de7-746a-3d7e-5c7c-57aa2e103a17/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T20:40:30,095 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50761 2024-11-26T20:40:30,108 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T20:40:30,111 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T20:40:30,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741825_1001 (size=7) 2024-11-26T20:40:30,789 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794 with version=8 2024-11-26T20:40:30,790 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/hbase-staging 2024-11-26T20:40:30,905 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-26T20:40:31,162 INFO [Time-limited test {}] client.ConnectionUtils(128): master/cd6be161543e:0 server-side Connection retries=45 2024-11-26T20:40:31,173 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:40:31,174 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T20:40:31,179 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T20:40:31,179 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:40:31,179 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T20:40:31,352 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T20:40:31,419 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-26T20:40:31,429 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-26T20:40:31,433 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T20:40:31,462 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1548 (auto-detected) 2024-11-26T20:40:31,464 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-26T20:40:31,492 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42359 2024-11-26T20:40:31,524 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42359 connecting to ZooKeeper ensemble=127.0.0.1:50761 2024-11-26T20:40:31,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:423590x0, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T20:40:31,574 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42359-0x10156022cfa0000 connected 2024-11-26T20:40:31,628 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T20:40:31,632 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T20:40:31,652 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:40:31,656 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794, hbase.cluster.distributed=false 2024-11-26T20:40:31,688 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T20:40:31,694 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42359 2024-11-26T20:40:31,695 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42359 2024-11-26T20:40:31,699 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42359 2024-11-26T20:40:31,701 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42359 2024-11-26T20:40:31,702 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42359 2024-11-26T20:40:31,844 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/cd6be161543e:0 server-side Connection retries=45 2024-11-26T20:40:31,847 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:40:31,847 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T20:40:31,847 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T20:40:31,848 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:40:31,848 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T20:40:31,851 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T20:40:31,862 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T20:40:31,863 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40291 2024-11-26T20:40:31,866 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40291 connecting to ZooKeeper ensemble=127.0.0.1:50761 2024-11-26T20:40:31,868 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T20:40:31,872 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T20:40:31,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:402910x0, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T20:40:31,885 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:402910x0, quorum=127.0.0.1:50761, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:40:31,886 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40291-0x10156022cfa0001 connected 2024-11-26T20:40:31,890 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T20:40:31,904 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T20:40:31,907 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40291-0x10156022cfa0001, quorum=127.0.0.1:50761, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T20:40:31,913 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40291-0x10156022cfa0001, quorum=127.0.0.1:50761, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T20:40:31,914 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40291 2024-11-26T20:40:31,914 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40291 2024-11-26T20:40:31,915 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40291 2024-11-26T20:40:31,916 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40291 2024-11-26T20:40:31,917 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40291 2024-11-26T20:40:31,939 DEBUG [M:0;cd6be161543e:42359 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;cd6be161543e:42359 2024-11-26T20:40:31,940 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/cd6be161543e,42359,1732653630959 2024-11-26T20:40:31,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:40:31,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40291-0x10156022cfa0001, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:40:31,952 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/cd6be161543e,42359,1732653630959 2024-11-26T20:40:31,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:40:31,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40291-0x10156022cfa0001, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T20:40:31,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40291-0x10156022cfa0001, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:40:31,979 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T20:40:31,981 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/cd6be161543e,42359,1732653630959 from backup master directory 2024-11-26T20:40:31,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/cd6be161543e,42359,1732653630959 2024-11-26T20:40:31,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:40:31,987 WARN [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T20:40:31,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40291-0x10156022cfa0001, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:40:31,987 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=cd6be161543e,42359,1732653630959 2024-11-26T20:40:31,990 INFO [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-26T20:40:31,993 INFO [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-26T20:40:32,068 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/hbase.id] with ID: de4c8973-112a-42f4-a492-8da7ceac9c1e 2024-11-26T20:40:32,068 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/.tmp/hbase.id 2024-11-26T20:40:32,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741826_1002 (size=42) 2024-11-26T20:40:32,484 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/.tmp/hbase.id]:[hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/hbase.id] 2024-11-26T20:40:32,536 INFO [master/cd6be161543e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T20:40:32,541 INFO [master/cd6be161543e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T20:40:32,565 INFO [master/cd6be161543e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 22ms. 2024-11-26T20:40:32,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:40:32,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40291-0x10156022cfa0001, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:40:32,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741827_1003 (size=196) 2024-11-26T20:40:32,794 INFO [master/cd6be161543e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T20:40:32,797 INFO [master/cd6be161543e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T20:40:32,819 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:32,825 INFO [master/cd6be161543e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T20:40:32,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741828_1004 (size=1189) 2024-11-26T20:40:32,877 INFO [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/data/master/store 2024-11-26T20:40:32,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741829_1005 (size=34) 2024-11-26T20:40:32,903 INFO [master/cd6be161543e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-26T20:40:32,907 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:40:32,908 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T20:40:32,908 INFO [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:40:32,909 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:40:32,910 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T20:40:32,911 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:40:32,911 INFO [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:40:32,912 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732653632908Disabling compacts and flushes for region at 1732653632908Disabling writes for close at 1732653632910 (+2 ms)Writing region close event to WAL at 1732653632911 (+1 ms)Closed at 1732653632911 2024-11-26T20:40:32,915 WARN [master/cd6be161543e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/data/master/store/.initializing 2024-11-26T20:40:32,915 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/WALs/cd6be161543e,42359,1732653630959 2024-11-26T20:40:32,926 INFO [master/cd6be161543e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T20:40:32,947 INFO [master/cd6be161543e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cd6be161543e%2C42359%2C1732653630959, suffix=, logDir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/WALs/cd6be161543e,42359,1732653630959, archiveDir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/oldWALs, maxLogs=10 2024-11-26T20:40:32,985 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/WALs/cd6be161543e,42359,1732653630959/cd6be161543e%2C42359%2C1732653630959.1732653632952, exclude list is [], retry=0 2024-11-26T20:40:33,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34989,DS-a4c67829-1b02-431d-831c-8acb2010147f,DISK] 2024-11-26T20:40:33,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-26T20:40:33,076 INFO [master/cd6be161543e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/WALs/cd6be161543e,42359,1732653630959/cd6be161543e%2C42359%2C1732653630959.1732653632952 2024-11-26T20:40:33,078 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38509:38509)] 2024-11-26T20:40:33,078 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:40:33,079 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:40:33,083 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:40:33,084 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:40:33,121 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:40:33,149 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T20:40:33,152 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:40:33,155 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:40:33,155 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:40:33,159 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T20:40:33,159 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:40:33,160 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:40:33,160 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:40:33,163 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T20:40:33,163 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:40:33,164 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:40:33,165 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:40:33,167 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T20:40:33,168 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:40:33,169 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:40:33,169 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:40:33,172 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:40:33,173 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:40:33,178 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:40:33,179 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:40:33,182 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T20:40:33,185 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:40:33,190 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:40:33,192 INFO [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66089530, jitterRate=-0.015189260244369507}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T20:40:33,198 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732653633096Initializing all the Stores at 1732653633099 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732653633099Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653633100 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653633101 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653633101Cleaning up temporary data from old regions at 1732653633179 (+78 ms)Region opened successfully at 1732653633198 (+19 ms) 2024-11-26T20:40:33,204 INFO [master/cd6be161543e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T20:40:33,245 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fe9447b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cd6be161543e/172.17.0.2:0 2024-11-26T20:40:33,285 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T20:40:33,298 INFO [master/cd6be161543e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T20:40:33,298 INFO [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T20:40:33,301 INFO [master/cd6be161543e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:40:33,303 INFO [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-26T20:40:33,308 INFO [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-26T20:40:33,308 INFO [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T20:40:33,341 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T20:40:33,353 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T20:40:33,356 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T20:40:33,359 INFO [master/cd6be161543e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T20:40:33,361 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T20:40:33,363 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T20:40:33,365 INFO [master/cd6be161543e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T20:40:33,368 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T20:40:33,370 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T20:40:33,371 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T20:40:33,373 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T20:40:33,391 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T20:40:33,393 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T20:40:33,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40291-0x10156022cfa0001, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T20:40:33,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T20:40:33,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40291-0x10156022cfa0001, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:40:33,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:40:33,401 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=cd6be161543e,42359,1732653630959, sessionid=0x10156022cfa0000, setting cluster-up flag (Was=false) 2024-11-26T20:40:33,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40291-0x10156022cfa0001, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:40:33,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:40:33,424 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T20:40:33,426 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=cd6be161543e,42359,1732653630959 2024-11-26T20:40:33,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:40:33,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40291-0x10156022cfa0001, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:40:33,443 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T20:40:33,445 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=cd6be161543e,42359,1732653630959 2024-11-26T20:40:33,453 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T20:40:33,524 INFO [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(746): ClusterId : de4c8973-112a-42f4-a492-8da7ceac9c1e 2024-11-26T20:40:33,528 DEBUG [RS:0;cd6be161543e:40291 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T20:40:33,547 DEBUG [RS:0;cd6be161543e:40291 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T20:40:33,547 DEBUG [RS:0;cd6be161543e:40291 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T20:40:33,551 DEBUG [RS:0;cd6be161543e:40291 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T20:40:33,552 DEBUG [RS:0;cd6be161543e:40291 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39d39628, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cd6be161543e/172.17.0.2:0 2024-11-26T20:40:33,555 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:40:33,567 INFO [master/cd6be161543e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T20:40:33,574 DEBUG [RS:0;cd6be161543e:40291 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;cd6be161543e:40291 2024-11-26T20:40:33,575 INFO [master/cd6be161543e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T20:40:33,579 INFO [RS:0;cd6be161543e:40291 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T20:40:33,579 INFO [RS:0;cd6be161543e:40291 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T20:40:33,579 DEBUG [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T20:40:33,582 INFO [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(2659): reportForDuty to master=cd6be161543e,42359,1732653630959 with port=40291, startcode=1732653631795 2024-11-26T20:40:33,581 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: cd6be161543e,42359,1732653630959 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T20:40:33,590 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/cd6be161543e:0, corePoolSize=5, maxPoolSize=5 2024-11-26T20:40:33,590 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/cd6be161543e:0, corePoolSize=5, maxPoolSize=5 2024-11-26T20:40:33,590 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/cd6be161543e:0, corePoolSize=5, maxPoolSize=5 2024-11-26T20:40:33,591 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/cd6be161543e:0, corePoolSize=5, maxPoolSize=5 2024-11-26T20:40:33,591 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/cd6be161543e:0, corePoolSize=10, maxPoolSize=10 2024-11-26T20:40:33,591 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:40:33,591 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/cd6be161543e:0, corePoolSize=2, maxPoolSize=2 2024-11-26T20:40:33,592 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:40:33,598 DEBUG [RS:0;cd6be161543e:40291 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T20:40:33,602 INFO [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732653663601 2024-11-26T20:40:33,602 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T20:40:33,603 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T20:40:33,603 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T20:40:33,604 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T20:40:33,609 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T20:40:33,610 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T20:40:33,610 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T20:40:33,610 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T20:40:33,611 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:40:33,611 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T20:40:33,613 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T20:40:33,639 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T20:40:33,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741831_1007 (size=1321) 2024-11-26T20:40:33,643 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T20:40:33,643 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T20:40:33,646 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T20:40:33,647 INFO [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T20:40:33,657 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/cd6be161543e:0:becomeActiveMaster-HFileCleaner.large.0-1732653633650,5,FailOnTimeoutGroup] 2024-11-26T20:40:33,659 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/cd6be161543e:0:becomeActiveMaster-HFileCleaner.small.0-1732653633657,5,FailOnTimeoutGroup] 2024-11-26T20:40:33,659 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T20:40:33,659 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T20:40:33,661 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56979, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T20:40:33,661 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T20:40:33,661 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T20:40:33,666 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42359 {}] master.ServerManager(363): Checking decommissioned status of RegionServer cd6be161543e,40291,1732653631795 2024-11-26T20:40:33,669 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42359 {}] master.ServerManager(517): Registering regionserver=cd6be161543e,40291,1732653631795 2024-11-26T20:40:33,685 DEBUG [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794 2024-11-26T20:40:33,685 DEBUG [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39285 2024-11-26T20:40:33,686 DEBUG [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T20:40:33,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T20:40:33,692 DEBUG [RS:0;cd6be161543e:40291 {}] zookeeper.ZKUtil(111): regionserver:40291-0x10156022cfa0001, quorum=127.0.0.1:50761, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/cd6be161543e,40291,1732653631795 2024-11-26T20:40:33,692 WARN [RS:0;cd6be161543e:40291 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T20:40:33,692 INFO [RS:0;cd6be161543e:40291 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T20:40:33,692 DEBUG [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/WALs/cd6be161543e,40291,1732653631795 2024-11-26T20:40:33,695 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [cd6be161543e,40291,1732653631795] 2024-11-26T20:40:33,728 INFO [RS:0;cd6be161543e:40291 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T20:40:33,746 INFO [RS:0;cd6be161543e:40291 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T20:40:33,752 INFO [RS:0;cd6be161543e:40291 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T20:40:33,752 INFO [RS:0;cd6be161543e:40291 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:40:33,753 INFO [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T20:40:33,760 INFO [RS:0;cd6be161543e:40291 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T20:40:33,761 INFO [RS:0;cd6be161543e:40291 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T20:40:33,762 DEBUG [RS:0;cd6be161543e:40291 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:40:33,762 DEBUG [RS:0;cd6be161543e:40291 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:40:33,762 DEBUG [RS:0;cd6be161543e:40291 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:40:33,762 DEBUG [RS:0;cd6be161543e:40291 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:40:33,763 DEBUG [RS:0;cd6be161543e:40291 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:40:33,763 DEBUG [RS:0;cd6be161543e:40291 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/cd6be161543e:0, corePoolSize=2, maxPoolSize=2 2024-11-26T20:40:33,763 DEBUG [RS:0;cd6be161543e:40291 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:40:33,763 DEBUG [RS:0;cd6be161543e:40291 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:40:33,764 DEBUG [RS:0;cd6be161543e:40291 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:40:33,764 DEBUG [RS:0;cd6be161543e:40291 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:40:33,764 DEBUG [RS:0;cd6be161543e:40291 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:40:33,764 DEBUG [RS:0;cd6be161543e:40291 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/cd6be161543e:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:40:33,766 DEBUG [RS:0;cd6be161543e:40291 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/cd6be161543e:0, corePoolSize=3, maxPoolSize=3 2024-11-26T20:40:33,766 DEBUG [RS:0;cd6be161543e:40291 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0, corePoolSize=3, maxPoolSize=3 2024-11-26T20:40:33,767 INFO [RS:0;cd6be161543e:40291 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T20:40:33,768 INFO [RS:0;cd6be161543e:40291 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T20:40:33,768 INFO [RS:0;cd6be161543e:40291 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:40:33,768 INFO [RS:0;cd6be161543e:40291 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T20:40:33,768 INFO [RS:0;cd6be161543e:40291 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T20:40:33,768 INFO [RS:0;cd6be161543e:40291 {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,40291,1732653631795-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T20:40:33,795 INFO [RS:0;cd6be161543e:40291 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T20:40:33,797 INFO [RS:0;cd6be161543e:40291 {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,40291,1732653631795-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:40:33,797 INFO [RS:0;cd6be161543e:40291 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:40:33,798 INFO [RS:0;cd6be161543e:40291 {}] regionserver.Replication(171): cd6be161543e,40291,1732653631795 started 2024-11-26T20:40:33,818 INFO [RS:0;cd6be161543e:40291 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:40:33,818 INFO [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(1482): Serving as cd6be161543e,40291,1732653631795, RpcServer on cd6be161543e/172.17.0.2:40291, sessionid=0x10156022cfa0001 2024-11-26T20:40:33,819 DEBUG [RS:0;cd6be161543e:40291 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T20:40:33,819 DEBUG [RS:0;cd6be161543e:40291 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager cd6be161543e,40291,1732653631795 2024-11-26T20:40:33,820 DEBUG [RS:0;cd6be161543e:40291 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cd6be161543e,40291,1732653631795' 2024-11-26T20:40:33,820 DEBUG [RS:0;cd6be161543e:40291 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T20:40:33,821 DEBUG [RS:0;cd6be161543e:40291 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T20:40:33,822 DEBUG [RS:0;cd6be161543e:40291 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T20:40:33,822 DEBUG [RS:0;cd6be161543e:40291 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T20:40:33,822 DEBUG [RS:0;cd6be161543e:40291 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager cd6be161543e,40291,1732653631795 2024-11-26T20:40:33,822 DEBUG [RS:0;cd6be161543e:40291 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cd6be161543e,40291,1732653631795' 2024-11-26T20:40:33,822 DEBUG [RS:0;cd6be161543e:40291 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T20:40:33,823 DEBUG [RS:0;cd6be161543e:40291 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T20:40:33,824 DEBUG [RS:0;cd6be161543e:40291 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T20:40:33,824 INFO [RS:0;cd6be161543e:40291 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T20:40:33,824 INFO [RS:0;cd6be161543e:40291 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T20:40:33,930 INFO [RS:0;cd6be161543e:40291 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T20:40:33,933 INFO [RS:0;cd6be161543e:40291 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cd6be161543e%2C40291%2C1732653631795, suffix=, logDir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/WALs/cd6be161543e,40291,1732653631795, archiveDir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/oldWALs, maxLogs=32 2024-11-26T20:40:33,950 DEBUG [RS:0;cd6be161543e:40291 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/WALs/cd6be161543e,40291,1732653631795/cd6be161543e%2C40291%2C1732653631795.1732653633935, exclude list is [], retry=0 2024-11-26T20:40:33,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34989,DS-a4c67829-1b02-431d-831c-8acb2010147f,DISK] 2024-11-26T20:40:33,957 INFO [RS:0;cd6be161543e:40291 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/WALs/cd6be161543e,40291,1732653631795/cd6be161543e%2C40291%2C1732653631795.1732653633935 2024-11-26T20:40:33,958 DEBUG [RS:0;cd6be161543e:40291 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38509:38509)] 2024-11-26T20:40:34,042 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T20:40:34,042 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794 2024-11-26T20:40:34,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741833_1009 (size=32) 2024-11-26T20:40:34,454 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:40:34,456 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T20:40:34,459 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T20:40:34,459 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:40:34,460 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:40:34,460 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T20:40:34,462 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T20:40:34,462 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:40:34,463 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:40:34,463 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T20:40:34,465 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T20:40:34,466 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:40:34,466 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:40:34,467 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T20:40:34,469 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T20:40:34,469 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:40:34,470 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:40:34,470 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T20:40:34,471 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740 2024-11-26T20:40:34,472 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740 2024-11-26T20:40:34,475 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T20:40:34,475 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T20:40:34,476 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T20:40:34,478 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T20:40:34,482 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:40:34,482 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75328027, jitterRate=0.12247507274150848}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T20:40:34,485 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732653634454Initializing all the Stores at 1732653634455 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732653634456 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732653634456Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653634456Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732653634456Cleaning up temporary data from old regions at 1732653634475 (+19 ms)Region opened successfully at 1732653634484 (+9 ms) 2024-11-26T20:40:34,485 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T20:40:34,485 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T20:40:34,485 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T20:40:34,485 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T20:40:34,485 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T20:40:34,486 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T20:40:34,487 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732653634485Disabling compacts and flushes for region at 1732653634485Disabling writes for close at 1732653634485Writing region close event to WAL at 1732653634486 (+1 ms)Closed at 1732653634486 2024-11-26T20:40:34,490 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T20:40:34,490 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T20:40:34,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T20:40:34,505 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:40:34,507 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T20:40:34,661 DEBUG [cd6be161543e:42359 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-26T20:40:34,673 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:40:34,680 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as cd6be161543e,40291,1732653631795, state=OPENING 2024-11-26T20:40:34,685 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T20:40:34,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:40:34,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40291-0x10156022cfa0001, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:40:34,688 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:40:34,689 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:40:34,689 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:40:34,691 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=cd6be161543e,40291,1732653631795}] 2024-11-26T20:40:34,866 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T20:40:34,870 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44345, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T20:40:34,881 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T20:40:34,882 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T20:40:34,882 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T20:40:34,885 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cd6be161543e%2C40291%2C1732653631795.meta, suffix=.meta, logDir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/WALs/cd6be161543e,40291,1732653631795, archiveDir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/oldWALs, maxLogs=32 2024-11-26T20:40:34,901 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/WALs/cd6be161543e,40291,1732653631795/cd6be161543e%2C40291%2C1732653631795.meta.1732653634887.meta, exclude list is [], retry=0 2024-11-26T20:40:34,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34989,DS-a4c67829-1b02-431d-831c-8acb2010147f,DISK] 2024-11-26T20:40:34,907 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/WALs/cd6be161543e,40291,1732653631795/cd6be161543e%2C40291%2C1732653631795.meta.1732653634887.meta 2024-11-26T20:40:34,908 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38509:38509)] 2024-11-26T20:40:34,908 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:40:34,910 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T20:40:34,912 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T20:40:34,917 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T20:40:34,921 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T20:40:34,922 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:40:34,922 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T20:40:34,922 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T20:40:34,925 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T20:40:34,927 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T20:40:34,927 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:40:34,928 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:40:34,928 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T20:40:34,930 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T20:40:34,930 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:40:34,931 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:40:34,931 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T20:40:34,932 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T20:40:34,933 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:40:34,933 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:40:34,933 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T20:40:34,935 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T20:40:34,935 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:40:34,935 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:40:34,936 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T20:40:34,937 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740 2024-11-26T20:40:34,939 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740 2024-11-26T20:40:34,942 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T20:40:34,942 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T20:40:34,943 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T20:40:34,945 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T20:40:34,947 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63479802, jitterRate=-0.05407723784446716}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T20:40:34,947 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T20:40:34,949 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732653634923Writing region info on filesystem at 1732653634923Initializing all the Stores at 1732653634925 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732653634925Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732653634925Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653634925Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732653634925Cleaning up temporary data from old regions at 1732653634942 (+17 ms)Running coprocessor post-open hooks at 1732653634947 (+5 ms)Region opened successfully at 1732653634948 (+1 ms) 2024-11-26T20:40:34,956 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732653634857 2024-11-26T20:40:34,968 DEBUG [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T20:40:34,968 INFO [RS_OPEN_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T20:40:34,970 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:40:34,972 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as cd6be161543e,40291,1732653631795, state=OPEN 2024-11-26T20:40:34,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:40:34,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40291-0x10156022cfa0001, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:40:34,981 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:40:34,981 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:40:34,982 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:34,986 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T20:40:34,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=cd6be161543e,40291,1732653631795 in 291 msec 2024-11-26T20:40:34,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T20:40:34,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 493 msec 2024-11-26T20:40:34,995 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T20:40:34,995 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T20:40:35,016 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:40:35,018 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:40:35,038 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:40:35,040 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35199, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:40:35,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5650 sec 2024-11-26T20:40:35,059 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732653635059, completionTime=-1 2024-11-26T20:40:35,061 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-26T20:40:35,061 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:40:35,089 INFO [master/cd6be161543e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-26T20:40:35,089 INFO [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732653695089 2024-11-26T20:40:35,089 INFO [master/cd6be161543e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732653755089 2024-11-26T20:40:35,089 INFO [master/cd6be161543e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-26T20:40:35,091 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,42359,1732653630959-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:40:35,091 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,42359,1732653630959-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:40:35,091 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,42359,1732653630959-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:40:35,093 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-cd6be161543e:42359, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:40:35,093 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T20:40:35,094 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T20:40:35,099 DEBUG [master/cd6be161543e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T20:40:35,125 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.137sec 2024-11-26T20:40:35,126 INFO [master/cd6be161543e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T20:40:35,128 INFO [master/cd6be161543e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T20:40:35,129 INFO [master/cd6be161543e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T20:40:35,129 INFO [master/cd6be161543e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T20:40:35,129 INFO [master/cd6be161543e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T20:40:35,130 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,42359,1732653630959-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T20:40:35,130 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,42359,1732653630959-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T20:40:35,138 DEBUG [master/cd6be161543e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T20:40:35,139 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T20:40:35,139 INFO [master/cd6be161543e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cd6be161543e,42359,1732653630959-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:40:35,234 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67d2c9bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:35,236 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-26T20:40:35,236 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-26T20:40:35,239 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:40:35,242 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:40:35,249 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:40:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:40:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:40:35,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25f18c81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:35,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:40:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:40:35,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:35,258 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:40:35,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a26ad16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:35,261 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:40:35,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:40:35,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:40:35,270 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40816, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:40:35,290 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=cd6be161543e,42359,1732653630959 2024-11-26T20:40:35,306 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=343, ProcessCount=11, AvailableMemoryMB=6533 2024-11-26T20:40:35,332 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T20:40:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is cd6be161543e,42359,1732653630959 2024-11-26T20:40:35,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@830d707 2024-11-26T20:40:35,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T20:40:35,346 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T20:40:35,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T20:40:35,353 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T20:40:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T20:40:35,368 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T20:40:35,370 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-26T20:40:35,371 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:40:35,373 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T20:40:35,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:40:35,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741835_1011 (size=1061) 2024-11-26T20:40:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:40:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:40:35,790 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794 2024-11-26T20:40:35,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741836_1012 (size=53) 2024-11-26T20:40:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:40:36,200 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:40:36,200 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f47d720597d62d3cb60d84298295c6f3, disabling compactions & flushes 2024-11-26T20:40:36,200 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:36,200 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:36,200 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. after waiting 0 ms 2024-11-26T20:40:36,200 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:36,200 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:36,200 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f47d720597d62d3cb60d84298295c6f3: Waiting for close lock at 1732653636200Disabling compacts and flushes for region at 1732653636200Disabling writes for close at 1732653636200Writing region close event to WAL at 1732653636200Closed at 1732653636200 2024-11-26T20:40:36,203 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T20:40:36,208 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732653636203"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732653636203"}]},"ts":"1732653636203"} 2024-11-26T20:40:36,213 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T20:40:36,215 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T20:40:36,218 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653636215"}]},"ts":"1732653636215"} 2024-11-26T20:40:36,222 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T20:40:36,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f47d720597d62d3cb60d84298295c6f3, ASSIGN}] 2024-11-26T20:40:36,226 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f47d720597d62d3cb60d84298295c6f3, ASSIGN 2024-11-26T20:40:36,227 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f47d720597d62d3cb60d84298295c6f3, ASSIGN; state=OFFLINE, location=cd6be161543e,40291,1732653631795; forceNewPlan=false, retain=false 2024-11-26T20:40:36,379 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f47d720597d62d3cb60d84298295c6f3, regionState=OPENING, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:40:36,383 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f47d720597d62d3cb60d84298295c6f3, ASSIGN because future has completed 2024-11-26T20:40:36,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795}] 2024-11-26T20:40:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:40:36,545 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:36,545 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:40:36,546 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:36,546 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:40:36,546 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:36,546 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:36,548 INFO [StoreOpener-f47d720597d62d3cb60d84298295c6f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:36,552 INFO [StoreOpener-f47d720597d62d3cb60d84298295c6f3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:40:36,552 INFO [StoreOpener-f47d720597d62d3cb60d84298295c6f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f47d720597d62d3cb60d84298295c6f3 columnFamilyName A 2024-11-26T20:40:36,552 DEBUG [StoreOpener-f47d720597d62d3cb60d84298295c6f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:40:36,553 INFO [StoreOpener-f47d720597d62d3cb60d84298295c6f3-1 {}] regionserver.HStore(327): Store=f47d720597d62d3cb60d84298295c6f3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:40:36,553 INFO [StoreOpener-f47d720597d62d3cb60d84298295c6f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:36,555 INFO [StoreOpener-f47d720597d62d3cb60d84298295c6f3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:40:36,555 INFO [StoreOpener-f47d720597d62d3cb60d84298295c6f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f47d720597d62d3cb60d84298295c6f3 columnFamilyName B 2024-11-26T20:40:36,556 DEBUG [StoreOpener-f47d720597d62d3cb60d84298295c6f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:40:36,556 INFO [StoreOpener-f47d720597d62d3cb60d84298295c6f3-1 {}] regionserver.HStore(327): Store=f47d720597d62d3cb60d84298295c6f3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:40:36,557 INFO [StoreOpener-f47d720597d62d3cb60d84298295c6f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:36,558 INFO [StoreOpener-f47d720597d62d3cb60d84298295c6f3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:40:36,559 INFO [StoreOpener-f47d720597d62d3cb60d84298295c6f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f47d720597d62d3cb60d84298295c6f3 columnFamilyName C 2024-11-26T20:40:36,559 DEBUG [StoreOpener-f47d720597d62d3cb60d84298295c6f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:40:36,560 INFO [StoreOpener-f47d720597d62d3cb60d84298295c6f3-1 {}] regionserver.HStore(327): Store=f47d720597d62d3cb60d84298295c6f3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:40:36,560 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:36,560 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:36,562 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:36,563 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:36,565 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:36,565 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:36,566 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T20:40:36,568 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:36,572 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:40:36,572 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f47d720597d62d3cb60d84298295c6f3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74251212, jitterRate=0.10642927885055542}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T20:40:36,572 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:36,573 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f47d720597d62d3cb60d84298295c6f3: Running coprocessor pre-open hook at 1732653636546Writing region info on filesystem at 1732653636546Initializing all the Stores at 1732653636548 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653636548Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653636548Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653636548Cleaning up temporary data from old regions at 1732653636565 (+17 ms)Running coprocessor post-open hooks at 1732653636572 (+7 ms)Region opened successfully at 1732653636573 (+1 ms) 2024-11-26T20:40:36,575 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., pid=6, masterSystemTime=1732653636538 2024-11-26T20:40:36,579 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:36,579 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:36,580 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f47d720597d62d3cb60d84298295c6f3, regionState=OPEN, openSeqNum=2, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:40:36,585 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 because future has completed 2024-11-26T20:40:36,590 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T20:40:36,590 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 in 203 msec 2024-11-26T20:40:36,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T20:40:36,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f47d720597d62d3cb60d84298295c6f3, ASSIGN in 367 msec 2024-11-26T20:40:36,596 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T20:40:36,597 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653636597"}]},"ts":"1732653636597"} 2024-11-26T20:40:36,600 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T20:40:36,601 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T20:40:36,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2430 sec 2024-11-26T20:40:37,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:40:37,525 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:40:37,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c8feb56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,531 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:40:37,531 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:40:37,533 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:40:37,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:40:37,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:40:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a4b96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:40:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:40:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:37,536 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:40:37,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185f36b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,538 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:40:37,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:40:37,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:40:37,542 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:40:37,544 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T20:40:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is cd6be161543e,42359,1732653630959 2024-11-26T20:40:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@25ca1f99 2024-11-26T20:40:37,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T20:40:37,549 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T20:40:37,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6432e8ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,554 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:40:37,554 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:40:37,555 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:40:37,555 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:40:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:40:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dd81cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:40:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:40:37,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:37,558 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:40:37,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9a6e4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@392f723b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,560 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:40:37,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:40:37,562 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:40:37,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:40:37,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:40:37,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89c0433, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:40:37,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:40:37,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:37,565 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:40:37,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f89bd3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0e98fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,567 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:40:37,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:40:37,568 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:40:37,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:40:37,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:40:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8ac8568, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:40:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:40:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:37,572 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:40:37,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab1aa4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69066494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,574 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:40:37,574 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:40:37,575 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:40:37,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:40:37,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:40:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f9668b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:40:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:40:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:37,578 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:40:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c1e7e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f00f692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,579 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:40:37,579 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:40:37,580 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:40:37,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:40:37,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:40:37,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cd9b0bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:40:37,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:40:37,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:37,582 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:40:37,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e801e27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ec8db7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,584 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:40:37,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:40:37,585 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:40:37,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:40:37,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:40:37,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bf4dbe1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:40:37,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:40:37,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:37,587 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:40:37,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66d1dde7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff89310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:40:37,588 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:40:37,589 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:40:37,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:40:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:40:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a88f812, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:40:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:40:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:37,591 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:40:37,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c7f9af0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14d81b23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,593 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:40:37,593 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:40:37,594 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:40:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:40:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:40:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53def629, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:40:37,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:40:37,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:37,596 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:40:37,597 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e3080a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2019371a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,597 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:40:37,597 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:40:37,598 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:40:37,599 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:40:37,599 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:40:37,599 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2268467d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,599 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:40:37,599 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:40:37,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:37,601 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:40:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0f145b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:40:37,607 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:40:37,607 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:40:37,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:40:37,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:40:37,610 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:40:37,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:40:37,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:40:37,611 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:40:37,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:40:37,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:40:37,613 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:40:37,614 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:40:37,614 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:40:37,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:40:37,616 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:40:37,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:40:37,616 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:40:37,617 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:40:37,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:40:37,618 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:40:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:40:37,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:40:37,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:40:37,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:40:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:40:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:40:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:40:37,623 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:40:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:40:37,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:40:37,626 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:40:37,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:40:37,628 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:40:37,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:40:37,630 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:40:37,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:40:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:40:37,632 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:40:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:40:37,634 DEBUG [Thread-158 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:40:37,634 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:40:37,634 DEBUG [Thread-158 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:40:37,636 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:40:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:40:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:40:37,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-26T20:40:37,639 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:40:37,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T20:40:37,641 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:40:37,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:40:37,644 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:40:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:37,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T20:40:37,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:37,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:37,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:37,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:37,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:37,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:37,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T20:40:37,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T20:40:37,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:37,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:37,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:37,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:37,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:37,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:37,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:37,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653697802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:37,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653697797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:37,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653697803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:37,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:37,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/4d29a0b523a44837a441ddcf994d60f4 is 50, key is test_row_0/A:col10/1732653637708/Put/seqid=0 2024-11-26T20:40:37,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653697820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:37,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:37,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653697821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:37,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741837_1013 (size=9657) 2024-11-26T20:40:37,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/4d29a0b523a44837a441ddcf994d60f4 2024-11-26T20:40:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T20:40:37,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653697965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:37,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653697965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:37,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/7c13c90fa60a476eb6f6a9edeb5f9e67 is 50, key is test_row_0/B:col10/1732653637708/Put/seqid=0 2024-11-26T20:40:37,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:37,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:37,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:37,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653697967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:37,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:37,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:37,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:37,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653697967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:37,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:37,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:37,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:37,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653697975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:37,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:37,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:37,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:37,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T20:40:37,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:37,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:37,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:37,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:37,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:37,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741838_1014 (size=9657) 2024-11-26T20:40:38,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/7c13c90fa60a476eb6f6a9edeb5f9e67 2024-11-26T20:40:38,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/22787a0caeb64567980af6e931953160 is 50, key is test_row_0/C:col10/1732653637708/Put/seqid=0 2024-11-26T20:40:38,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741839_1015 (size=9657) 2024-11-26T20:40:38,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/22787a0caeb64567980af6e931953160 2024-11-26T20:40:38,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/4d29a0b523a44837a441ddcf994d60f4 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/4d29a0b523a44837a441ddcf994d60f4 2024-11-26T20:40:38,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/4d29a0b523a44837a441ddcf994d60f4, entries=100, sequenceid=13, filesize=9.4 K 2024-11-26T20:40:38,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/7c13c90fa60a476eb6f6a9edeb5f9e67 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/7c13c90fa60a476eb6f6a9edeb5f9e67 2024-11-26T20:40:38,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/7c13c90fa60a476eb6f6a9edeb5f9e67, entries=100, sequenceid=13, filesize=9.4 K 2024-11-26T20:40:38,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/22787a0caeb64567980af6e931953160 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/22787a0caeb64567980af6e931953160 2024-11-26T20:40:38,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/22787a0caeb64567980af6e931953160, entries=100, sequenceid=13, filesize=9.4 K 2024-11-26T20:40:38,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f47d720597d62d3cb60d84298295c6f3 in 416ms, sequenceid=13, compaction requested=false 2024-11-26T20:40:38,136 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-26T20:40:38,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:38,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T20:40:38,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:38,148 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T20:40:38,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:38,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:38,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:38,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:38,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:38,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:38,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/9eaab4d6f05a4df49904b3e3a9e53a24 is 50, key is test_row_0/A:col10/1732653637795/Put/seqid=0 2024-11-26T20:40:38,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:38,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:38,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741840_1016 (size=12001) 2024-11-26T20:40:38,198 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/9eaab4d6f05a4df49904b3e3a9e53a24 2024-11-26T20:40:38,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:38,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653698213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:38,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:38,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:38,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653698213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:38,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/3b8351a31f354ff89d2d604e4ddf3ea8 is 50, key is test_row_0/B:col10/1732653637795/Put/seqid=0 2024-11-26T20:40:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:38,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:38,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653698221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:38,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:38,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653698217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:38,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653698226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:38,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:38,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741841_1017 (size=12001) 2024-11-26T20:40:38,260 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/3b8351a31f354ff89d2d604e4ddf3ea8 2024-11-26T20:40:38,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T20:40:38,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/a325a8a70d2b417b9eed2e9a864a2326 is 50, key is test_row_0/C:col10/1732653637795/Put/seqid=0 2024-11-26T20:40:38,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741842_1018 (size=12001) 2024-11-26T20:40:38,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:38,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653698335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:38,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653698335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:38,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:38,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653698345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:38,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:38,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653698345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:38,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:38,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:38,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653698346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:38,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:38,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653698546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:38,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:38,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653698546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:38,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653698555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:38,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653698555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:38,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:38,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:38,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653698568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:38,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:38,725 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/a325a8a70d2b417b9eed2e9a864a2326 2024-11-26T20:40:38,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/9eaab4d6f05a4df49904b3e3a9e53a24 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/9eaab4d6f05a4df49904b3e3a9e53a24 2024-11-26T20:40:38,761 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/9eaab4d6f05a4df49904b3e3a9e53a24, entries=150, sequenceid=33, filesize=11.7 K 2024-11-26T20:40:38,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/3b8351a31f354ff89d2d604e4ddf3ea8 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/3b8351a31f354ff89d2d604e4ddf3ea8 2024-11-26T20:40:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T20:40:38,780 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/3b8351a31f354ff89d2d604e4ddf3ea8, entries=150, sequenceid=33, filesize=11.7 K 2024-11-26T20:40:38,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/a325a8a70d2b417b9eed2e9a864a2326 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/a325a8a70d2b417b9eed2e9a864a2326 2024-11-26T20:40:38,800 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/a325a8a70d2b417b9eed2e9a864a2326, entries=150, sequenceid=33, filesize=11.7 K 2024-11-26T20:40:38,805 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f47d720597d62d3cb60d84298295c6f3 in 658ms, sequenceid=33, compaction requested=false 2024-11-26T20:40:38,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:38,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:38,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-26T20:40:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-26T20:40:38,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-26T20:40:38,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1650 sec 2024-11-26T20:40:38,822 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.2010 sec 2024-11-26T20:40:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:38,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T20:40:38,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:38,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:38,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:38,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:38,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:38,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:38,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/4c477350e95f473bb56f7b7a785d0b0d is 50, key is test_row_0/A:col10/1732653638858/Put/seqid=0 2024-11-26T20:40:38,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741843_1019 (size=14341) 2024-11-26T20:40:38,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/4c477350e95f473bb56f7b7a785d0b0d 2024-11-26T20:40:38,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:38,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653698934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653698936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:38,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:38,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:38,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653698939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:38,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:38,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:38,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:38,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653698946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:38,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:38,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:38,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653698949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:38,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/21b99bbff1e343fc8d53cdd55a99193f is 50, key is test_row_0/B:col10/1732653638858/Put/seqid=0 2024-11-26T20:40:38,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741844_1020 (size=12001) 2024-11-26T20:40:38,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/21b99bbff1e343fc8d53cdd55a99193f 2024-11-26T20:40:39,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/bca845eaef9f4f5d9c3fe25bce0adf2b is 50, key is test_row_0/C:col10/1732653638858/Put/seqid=0 2024-11-26T20:40:39,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741845_1021 (size=12001) 2024-11-26T20:40:39,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/bca845eaef9f4f5d9c3fe25bce0adf2b 2024-11-26T20:40:39,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/4c477350e95f473bb56f7b7a785d0b0d as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/4c477350e95f473bb56f7b7a785d0b0d 2024-11-26T20:40:39,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:39,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653699055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:39,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:39,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:39,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653699056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:39,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:39,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:39,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653699056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:39,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:39,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/4c477350e95f473bb56f7b7a785d0b0d, entries=200, sequenceid=47, filesize=14.0 K 2024-11-26T20:40:39,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/21b99bbff1e343fc8d53cdd55a99193f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/21b99bbff1e343fc8d53cdd55a99193f 2024-11-26T20:40:39,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653699066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:39,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:39,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653699066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:39,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:39,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/21b99bbff1e343fc8d53cdd55a99193f, entries=150, sequenceid=47, filesize=11.7 K 2024-11-26T20:40:39,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/bca845eaef9f4f5d9c3fe25bce0adf2b as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/bca845eaef9f4f5d9c3fe25bce0adf2b 2024-11-26T20:40:39,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/bca845eaef9f4f5d9c3fe25bce0adf2b, entries=150, sequenceid=47, filesize=11.7 K 2024-11-26T20:40:39,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f47d720597d62d3cb60d84298295c6f3 in 250ms, sequenceid=47, compaction requested=true 2024-11-26T20:40:39,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:39,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:40:39,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:39,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:40:39,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:39,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:40:39,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:39,130 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:39,131 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:39,134 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:39,134 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:39,137 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/A is initiating minor compaction (all files) 2024-11-26T20:40:39,137 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/B is initiating minor compaction (all files) 2024-11-26T20:40:39,138 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/B in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:39,138 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/A in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:39,139 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/4d29a0b523a44837a441ddcf994d60f4, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/9eaab4d6f05a4df49904b3e3a9e53a24, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/4c477350e95f473bb56f7b7a785d0b0d] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=35.2 K 2024-11-26T20:40:39,139 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/7c13c90fa60a476eb6f6a9edeb5f9e67, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/3b8351a31f354ff89d2d604e4ddf3ea8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/21b99bbff1e343fc8d53cdd55a99193f] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=32.9 K 2024-11-26T20:40:39,140 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d29a0b523a44837a441ddcf994d60f4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732653637682 2024-11-26T20:40:39,140 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c13c90fa60a476eb6f6a9edeb5f9e67, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732653637682 2024-11-26T20:40:39,142 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b8351a31f354ff89d2d604e4ddf3ea8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732653637795 2024-11-26T20:40:39,146 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 21b99bbff1e343fc8d53cdd55a99193f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732653638220 2024-11-26T20:40:39,146 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9eaab4d6f05a4df49904b3e3a9e53a24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732653637795 2024-11-26T20:40:39,147 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c477350e95f473bb56f7b7a785d0b0d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732653638204 2024-11-26T20:40:39,230 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#B#compaction#9 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:39,231 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/5494621c2c0942d1b1931e17e3c00387 is 50, key is test_row_0/B:col10/1732653638858/Put/seqid=0 2024-11-26T20:40:39,243 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#A#compaction#10 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:39,244 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/4f59f5c9dd8041fbb21d3fda63fd58d6 is 50, key is test_row_0/A:col10/1732653638858/Put/seqid=0 2024-11-26T20:40:39,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741846_1022 (size=12104) 2024-11-26T20:40:39,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T20:40:39,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:39,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:39,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:39,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:39,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:39,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:39,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741847_1023 (size=12104) 2024-11-26T20:40:39,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/de7ec32aa945426795eb33c4120a66da is 50, key is test_row_0/A:col10/1732653639283/Put/seqid=0 2024-11-26T20:40:39,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653699320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:39,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741848_1024 (size=12001) 2024-11-26T20:40:39,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:39,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653699327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:39,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:39,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653699399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:39,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653699402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:39,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653699403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:39,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:39,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653699515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:39,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:39,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:39,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653699515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653699515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:39,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653699516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:39,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:39,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:39,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653699525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:39,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:39,679 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/5494621c2c0942d1b1931e17e3c00387 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/5494621c2c0942d1b1931e17e3c00387 2024-11-26T20:40:39,719 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T20:40:39,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:39,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653699724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:39,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:39,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653699736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:39,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653699737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:39,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653699738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:39,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653699738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:39,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:39,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:39,757 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/4f59f5c9dd8041fbb21d3fda63fd58d6 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/4f59f5c9dd8041fbb21d3fda63fd58d6 2024-11-26T20:40:39,778 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/A of f47d720597d62d3cb60d84298295c6f3 into 4f59f5c9dd8041fbb21d3fda63fd58d6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:39,779 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:39,784 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/A, priority=13, startTime=1732653639112; duration=0sec 2024-11-26T20:40:39,784 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/B of f47d720597d62d3cb60d84298295c6f3 into 5494621c2c0942d1b1931e17e3c00387(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:39,784 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:39,784 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/B, priority=13, startTime=1732653639128; duration=0sec 2024-11-26T20:40:39,784 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:39,785 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:A 2024-11-26T20:40:39,785 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:39,785 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:B 2024-11-26T20:40:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T20:40:39,785 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:39,785 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:40:39,787 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:40:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-26T20:40:39,791 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:39,791 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/C is initiating minor compaction (all files) 2024-11-26T20:40:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T20:40:39,791 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/C in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:39,791 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/22787a0caeb64567980af6e931953160, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/a325a8a70d2b417b9eed2e9a864a2326, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/bca845eaef9f4f5d9c3fe25bce0adf2b] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=32.9 K 2024-11-26T20:40:39,792 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:40:39,793 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 22787a0caeb64567980af6e931953160, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732653637682 2024-11-26T20:40:39,795 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting a325a8a70d2b417b9eed2e9a864a2326, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732653637795 2024-11-26T20:40:39,795 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:40:39,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:40:39,796 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting bca845eaef9f4f5d9c3fe25bce0adf2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732653638220 2024-11-26T20:40:39,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/de7ec32aa945426795eb33c4120a66da 2024-11-26T20:40:39,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/54c53dd1aa7e40009f5b223b2747db35 is 50, key is test_row_0/B:col10/1732653639283/Put/seqid=0 2024-11-26T20:40:39,850 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#C#compaction#13 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:39,851 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/b8a028dfc12b42ffb7208dd8a2dcd250 is 50, key is test_row_0/C:col10/1732653638858/Put/seqid=0 2024-11-26T20:40:39,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741849_1025 (size=12001) 2024-11-26T20:40:39,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/54c53dd1aa7e40009f5b223b2747db35 2024-11-26T20:40:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T20:40:39,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741850_1026 (size=12104) 2024-11-26T20:40:39,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/bcad5ecac0954d25897d7a18d2cea7fb is 50, key is test_row_0/C:col10/1732653639283/Put/seqid=0 2024-11-26T20:40:39,928 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/b8a028dfc12b42ffb7208dd8a2dcd250 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/b8a028dfc12b42ffb7208dd8a2dcd250 2024-11-26T20:40:39,942 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/C of f47d720597d62d3cb60d84298295c6f3 into b8a028dfc12b42ffb7208dd8a2dcd250(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:39,942 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:39,943 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/C, priority=13, startTime=1732653639128; duration=0sec 2024-11-26T20:40:39,943 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:39,943 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:C 2024-11-26T20:40:39,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741851_1027 (size=12001) 2024-11-26T20:40:39,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/bcad5ecac0954d25897d7a18d2cea7fb 2024-11-26T20:40:39,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T20:40:39,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:39,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:39,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:39,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:39,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:39,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/de7ec32aa945426795eb33c4120a66da as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/de7ec32aa945426795eb33c4120a66da 2024-11-26T20:40:39,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/de7ec32aa945426795eb33c4120a66da, entries=150, sequenceid=67, filesize=11.7 K 2024-11-26T20:40:39,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/54c53dd1aa7e40009f5b223b2747db35 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/54c53dd1aa7e40009f5b223b2747db35 2024-11-26T20:40:39,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/54c53dd1aa7e40009f5b223b2747db35, entries=150, sequenceid=67, filesize=11.7 K 2024-11-26T20:40:40,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/bcad5ecac0954d25897d7a18d2cea7fb as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/bcad5ecac0954d25897d7a18d2cea7fb 2024-11-26T20:40:40,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/bcad5ecac0954d25897d7a18d2cea7fb, entries=150, sequenceid=67, filesize=11.7 K 2024-11-26T20:40:40,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f47d720597d62d3cb60d84298295c6f3 in 738ms, sequenceid=67, compaction requested=false 2024-11-26T20:40:40,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:40,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T20:40:40,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:40,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:40,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:40,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:40,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:40,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:40,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/293567408a58489394fc540beb11df43 is 50, key is test_row_0/A:col10/1732653639331/Put/seqid=0 2024-11-26T20:40:40,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741852_1028 (size=14341) 2024-11-26T20:40:40,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/293567408a58489394fc540beb11df43 2024-11-26T20:40:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T20:40:40,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T20:40:40,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:40,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:40,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:40,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:40,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:40,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653700097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653700100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653700101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653700105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653700111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/f5ae4432a7f84606bf4343409d9ff292 is 50, key is test_row_0/B:col10/1732653639331/Put/seqid=0 2024-11-26T20:40:40,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741853_1029 (size=12001) 2024-11-26T20:40:40,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/f5ae4432a7f84606bf4343409d9ff292 2024-11-26T20:40:40,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/20a949829ed14d0292e436b308297196 is 50, key is test_row_0/C:col10/1732653639331/Put/seqid=0 2024-11-26T20:40:40,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653700225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653700225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653700225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653700226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653700226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741854_1030 (size=12001) 2024-11-26T20:40:40,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/20a949829ed14d0292e436b308297196 2024-11-26T20:40:40,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/293567408a58489394fc540beb11df43 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/293567408a58489394fc540beb11df43 2024-11-26T20:40:40,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/293567408a58489394fc540beb11df43, entries=200, sequenceid=84, filesize=14.0 K 2024-11-26T20:40:40,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/f5ae4432a7f84606bf4343409d9ff292 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/f5ae4432a7f84606bf4343409d9ff292 2024-11-26T20:40:40,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T20:40:40,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:40,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:40,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:40,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:40,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:40,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/f5ae4432a7f84606bf4343409d9ff292, entries=150, sequenceid=84, filesize=11.7 K 2024-11-26T20:40:40,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/20a949829ed14d0292e436b308297196 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/20a949829ed14d0292e436b308297196 2024-11-26T20:40:40,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/20a949829ed14d0292e436b308297196, entries=150, sequenceid=84, filesize=11.7 K 2024-11-26T20:40:40,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f47d720597d62d3cb60d84298295c6f3 in 240ms, sequenceid=84, compaction requested=true 2024-11-26T20:40:40,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:40,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:40:40,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:40,295 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:40,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:40:40,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:40,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:40:40,295 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:40,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:40,298 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:40,298 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/A is initiating minor compaction (all files) 2024-11-26T20:40:40,298 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/A in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:40,298 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/4f59f5c9dd8041fbb21d3fda63fd58d6, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/de7ec32aa945426795eb33c4120a66da, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/293567408a58489394fc540beb11df43] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=37.5 K 2024-11-26T20:40:40,299 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:40,299 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f59f5c9dd8041fbb21d3fda63fd58d6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732653638220 2024-11-26T20:40:40,299 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/B is initiating minor compaction (all files) 2024-11-26T20:40:40,299 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/B in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:40,299 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/5494621c2c0942d1b1931e17e3c00387, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/54c53dd1aa7e40009f5b223b2747db35, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/f5ae4432a7f84606bf4343409d9ff292] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=35.3 K 2024-11-26T20:40:40,300 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting de7ec32aa945426795eb33c4120a66da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732653638927 2024-11-26T20:40:40,300 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 5494621c2c0942d1b1931e17e3c00387, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732653638220 2024-11-26T20:40:40,300 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 293567408a58489394fc540beb11df43, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732653639307 2024-11-26T20:40:40,301 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 54c53dd1aa7e40009f5b223b2747db35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732653638927 2024-11-26T20:40:40,302 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting f5ae4432a7f84606bf4343409d9ff292, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732653639307 2024-11-26T20:40:40,336 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#B#compaction#18 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:40,337 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/c3723845162d44a593bce0036d6c4aa9 is 50, key is test_row_0/B:col10/1732653639331/Put/seqid=0 2024-11-26T20:40:40,361 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#A#compaction#19 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:40,362 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/b4de1e1eec514ba686918ad646144ca9 is 50, key is test_row_0/A:col10/1732653639331/Put/seqid=0 2024-11-26T20:40:40,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741855_1031 (size=12207) 2024-11-26T20:40:40,405 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/c3723845162d44a593bce0036d6c4aa9 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/c3723845162d44a593bce0036d6c4aa9 2024-11-26T20:40:40,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741856_1032 (size=12207) 2024-11-26T20:40:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T20:40:40,423 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/b4de1e1eec514ba686918ad646144ca9 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/b4de1e1eec514ba686918ad646144ca9 2024-11-26T20:40:40,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T20:40:40,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:40,426 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T20:40:40,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:40,427 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/B of f47d720597d62d3cb60d84298295c6f3 into c3723845162d44a593bce0036d6c4aa9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:40,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:40,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:40,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:40,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:40,427 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:40,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:40,427 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/B, priority=13, startTime=1732653640295; duration=0sec 2024-11-26T20:40:40,429 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:40,429 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:B 2024-11-26T20:40:40,429 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:40,433 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:40,433 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/C is initiating minor compaction (all files) 2024-11-26T20:40:40,433 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/C in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:40,435 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/b8a028dfc12b42ffb7208dd8a2dcd250, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/bcad5ecac0954d25897d7a18d2cea7fb, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/20a949829ed14d0292e436b308297196] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=35.3 K 2024-11-26T20:40:40,436 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting b8a028dfc12b42ffb7208dd8a2dcd250, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732653638220 2024-11-26T20:40:40,437 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting bcad5ecac0954d25897d7a18d2cea7fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732653638927 2024-11-26T20:40:40,437 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/A of f47d720597d62d3cb60d84298295c6f3 into b4de1e1eec514ba686918ad646144ca9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:40,437 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:40,437 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/A, priority=13, startTime=1732653640294; duration=0sec 2024-11-26T20:40:40,437 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:40,438 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 20a949829ed14d0292e436b308297196, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732653639307 2024-11-26T20:40:40,438 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:A 2024-11-26T20:40:40,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/e8791a965c764f93956b8636903bc09c is 50, key is test_row_0/A:col10/1732653640096/Put/seqid=0 2024-11-26T20:40:40,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:40,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:40,456 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T20:40:40,485 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:40,487 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/f967319e74fe47ed82e3af9f4f3372c6 is 50, key is test_row_0/C:col10/1732653639331/Put/seqid=0 2024-11-26T20:40:40,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653700486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653700490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653700496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653700502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653700502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741857_1033 (size=12001) 2024-11-26T20:40:40,520 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/e8791a965c764f93956b8636903bc09c 2024-11-26T20:40:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/be03102a6eb34b7abbaccd33d169bff5 is 50, key is test_row_0/B:col10/1732653640096/Put/seqid=0 2024-11-26T20:40:40,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741858_1034 (size=12207) 2024-11-26T20:40:40,571 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/f967319e74fe47ed82e3af9f4f3372c6 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/f967319e74fe47ed82e3af9f4f3372c6 2024-11-26T20:40:40,586 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/C of f47d720597d62d3cb60d84298295c6f3 into f967319e74fe47ed82e3af9f4f3372c6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:40,586 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:40,587 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/C, priority=13, startTime=1732653640295; duration=0sec 2024-11-26T20:40:40,587 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:40,587 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:C 2024-11-26T20:40:40,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741859_1035 (size=12001) 2024-11-26T20:40:40,606 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/be03102a6eb34b7abbaccd33d169bff5 2024-11-26T20:40:40,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653700615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653700615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653700626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653700626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653700625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/1db4b4634cd8406e9236da79767f714d is 50, key is test_row_0/C:col10/1732653640096/Put/seqid=0 2024-11-26T20:40:40,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741860_1036 (size=12001) 2024-11-26T20:40:40,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653700824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653700835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653700835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653700835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653700836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:40,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T20:40:41,083 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/1db4b4634cd8406e9236da79767f714d 2024-11-26T20:40:41,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/e8791a965c764f93956b8636903bc09c as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/e8791a965c764f93956b8636903bc09c 2024-11-26T20:40:41,114 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/e8791a965c764f93956b8636903bc09c, entries=150, sequenceid=103, filesize=11.7 K 2024-11-26T20:40:41,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/be03102a6eb34b7abbaccd33d169bff5 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/be03102a6eb34b7abbaccd33d169bff5 2024-11-26T20:40:41,141 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/be03102a6eb34b7abbaccd33d169bff5, entries=150, sequenceid=103, filesize=11.7 K 2024-11-26T20:40:41,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/1db4b4634cd8406e9236da79767f714d as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/1db4b4634cd8406e9236da79767f714d 2024-11-26T20:40:41,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:41,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653701145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:41,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:41,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:41,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:41,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:41,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653701145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:41,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:41,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:41,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653701145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:41,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:41,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:41,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:41,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:41,158 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/1db4b4634cd8406e9236da79767f714d, entries=150, sequenceid=103, filesize=11.7 K 2024-11-26T20:40:41,160 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for f47d720597d62d3cb60d84298295c6f3 in 734ms, sequenceid=103, compaction requested=false 2024-11-26T20:40:41,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:41,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:41,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-26T20:40:41,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-26T20:40:41,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653701145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:41,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:41,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:41,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:41,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-26T20:40:41,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3680 sec 2024-11-26T20:40:41,181 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.3810 sec 2024-11-26T20:40:41,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:41,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T20:40:41,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:41,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:41,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:41,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:41,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:41,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:41,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/0578b0f01d6c456d820c536eb5077908 is 50, key is test_row_0/A:col10/1732653640496/Put/seqid=0 2024-11-26T20:40:41,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741861_1037 (size=14341) 2024-11-26T20:40:41,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653701316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:41,414 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T20:40:41,415 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-26T20:40:41,418 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T20:40:41,418 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-26T20:40:41,419 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-26T20:40:41,419 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-26T20:40:41,421 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-26T20:40:41,421 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-26T20:40:41,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653701430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:41,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:41,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:41,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:41,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/0578b0f01d6c456d820c536eb5077908 2024-11-26T20:40:41,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:41,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653701648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:41,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:41,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653701655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:41,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:41,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653701655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:41,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:41,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653701664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:41,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:41,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:41,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:41,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/c27fb7b6857c45e9b87d3952b89a16f6 is 50, key is test_row_0/B:col10/1732653640496/Put/seqid=0 2024-11-26T20:40:41,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:41,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653701675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:41,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741862_1038 (size=12001) 2024-11-26T20:40:41,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/c27fb7b6857c45e9b87d3952b89a16f6 2024-11-26T20:40:41,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/8aa778824197454ab7210dd6490d514f is 50, key is test_row_0/C:col10/1732653640496/Put/seqid=0 2024-11-26T20:40:41,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741863_1039 (size=12001) 2024-11-26T20:40:41,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/8aa778824197454ab7210dd6490d514f 2024-11-26T20:40:41,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/0578b0f01d6c456d820c536eb5077908 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/0578b0f01d6c456d820c536eb5077908 2024-11-26T20:40:41,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/0578b0f01d6c456d820c536eb5077908, entries=200, sequenceid=121, filesize=14.0 K 2024-11-26T20:40:41,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/c27fb7b6857c45e9b87d3952b89a16f6 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/c27fb7b6857c45e9b87d3952b89a16f6 2024-11-26T20:40:41,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/c27fb7b6857c45e9b87d3952b89a16f6, entries=150, sequenceid=121, filesize=11.7 K 2024-11-26T20:40:41,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/8aa778824197454ab7210dd6490d514f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/8aa778824197454ab7210dd6490d514f 2024-11-26T20:40:41,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/8aa778824197454ab7210dd6490d514f, entries=150, sequenceid=121, filesize=11.7 K 2024-11-26T20:40:41,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for f47d720597d62d3cb60d84298295c6f3 in 679ms, sequenceid=121, compaction requested=true 2024-11-26T20:40:41,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:41,865 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:41,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:40:41,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:41,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:40:41,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:41,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:40:41,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:41,866 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:41,868 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:41,869 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/A is initiating minor compaction (all files) 2024-11-26T20:40:41,869 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:41,869 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/A in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:41,869 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/B is initiating minor compaction (all files) 2024-11-26T20:40:41,869 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/B in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:41,869 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/b4de1e1eec514ba686918ad646144ca9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/e8791a965c764f93956b8636903bc09c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/0578b0f01d6c456d820c536eb5077908] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=37.6 K 2024-11-26T20:40:41,869 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/c3723845162d44a593bce0036d6c4aa9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/be03102a6eb34b7abbaccd33d169bff5, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/c27fb7b6857c45e9b87d3952b89a16f6] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=35.4 K 2024-11-26T20:40:41,870 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3723845162d44a593bce0036d6c4aa9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732653639307 2024-11-26T20:40:41,870 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting b4de1e1eec514ba686918ad646144ca9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732653639307 2024-11-26T20:40:41,870 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting e8791a965c764f93956b8636903bc09c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732653640096 2024-11-26T20:40:41,871 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting be03102a6eb34b7abbaccd33d169bff5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732653640096 2024-11-26T20:40:41,871 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 0578b0f01d6c456d820c536eb5077908, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732653640482 2024-11-26T20:40:41,872 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting c27fb7b6857c45e9b87d3952b89a16f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732653640491 2024-11-26T20:40:41,898 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#B#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:41,899 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/24b5cbe4151a4f4fa05ba8089aa9d086 is 50, key is test_row_0/B:col10/1732653640496/Put/seqid=0 2024-11-26T20:40:41,903 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#A#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:41,904 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/e7d83af8a1c649f3a7c32a04372534b7 is 50, key is test_row_0/A:col10/1732653640496/Put/seqid=0 2024-11-26T20:40:41,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741864_1040 (size=12309) 2024-11-26T20:40:41,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741865_1041 (size=12309) 2024-11-26T20:40:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T20:40:41,935 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:40:41,936 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:40:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-26T20:40:41,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T20:40:41,939 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:40:41,941 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:40:41,941 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:40:41,946 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/e7d83af8a1c649f3a7c32a04372534b7 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/e7d83af8a1c649f3a7c32a04372534b7 2024-11-26T20:40:41,962 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/A of f47d720597d62d3cb60d84298295c6f3 into e7d83af8a1c649f3a7c32a04372534b7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:41,962 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:41,962 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/A, priority=13, startTime=1732653641865; duration=0sec 2024-11-26T20:40:41,962 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:41,962 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:A 2024-11-26T20:40:41,962 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:41,987 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:41,987 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/C is initiating minor compaction (all files) 2024-11-26T20:40:41,987 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/C in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:41,988 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/f967319e74fe47ed82e3af9f4f3372c6, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/1db4b4634cd8406e9236da79767f714d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/8aa778824197454ab7210dd6490d514f] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=35.4 K 2024-11-26T20:40:41,991 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting f967319e74fe47ed82e3af9f4f3372c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732653639307 2024-11-26T20:40:41,992 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 1db4b4634cd8406e9236da79767f714d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732653640096 2024-11-26T20:40:41,993 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 8aa778824197454ab7210dd6490d514f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732653640491 2024-11-26T20:40:42,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T20:40:42,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:42,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:42,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:42,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:42,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:42,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:42,021 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#C#compaction#29 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:42,022 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/9e3dc9f2d29246768d1b02bfdce972fc is 50, key is test_row_0/C:col10/1732653640496/Put/seqid=0 2024-11-26T20:40:42,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/d6436d81097f458cb4477968631a4712 is 50, key is test_row_0/A:col10/1732653642002/Put/seqid=0 2024-11-26T20:40:42,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T20:40:42,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741866_1042 (size=12309) 2024-11-26T20:40:42,079 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/9e3dc9f2d29246768d1b02bfdce972fc as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/9e3dc9f2d29246768d1b02bfdce972fc 2024-11-26T20:40:42,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741867_1043 (size=14541) 2024-11-26T20:40:42,089 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/C of f47d720597d62d3cb60d84298295c6f3 into 9e3dc9f2d29246768d1b02bfdce972fc(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:42,089 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:42,089 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/C, priority=13, startTime=1732653641866; duration=0sec 2024-11-26T20:40:42,089 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:42,089 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:C 2024-11-26T20:40:42,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/d6436d81097f458cb4477968631a4712 2024-11-26T20:40:42,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T20:40:42,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:42,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:42,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:42,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:42,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:42,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:42,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/cce7820acd5a4fb2b208f25079a950b5 is 50, key is test_row_0/B:col10/1732653642002/Put/seqid=0 2024-11-26T20:40:42,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:42,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741868_1044 (size=12151) 2024-11-26T20:40:42,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653702119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:42,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:42,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/cce7820acd5a4fb2b208f25079a950b5 2024-11-26T20:40:42,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/362580f0d5cc4010a70a96e7ca2f8f5e is 50, key is test_row_0/C:col10/1732653642002/Put/seqid=0 2024-11-26T20:40:42,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741869_1045 (size=12151) 2024-11-26T20:40:42,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:42,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653702234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:42,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T20:40:42,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:42,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:42,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:42,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:42,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:42,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T20:40:42,339 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/24b5cbe4151a4f4fa05ba8089aa9d086 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/24b5cbe4151a4f4fa05ba8089aa9d086 2024-11-26T20:40:42,351 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/B of f47d720597d62d3cb60d84298295c6f3 into 24b5cbe4151a4f4fa05ba8089aa9d086(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:42,352 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:42,352 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/B, priority=13, startTime=1732653641866; duration=0sec 2024-11-26T20:40:42,353 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:42,353 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:B 2024-11-26T20:40:42,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T20:40:42,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:42,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:42,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:42,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:42,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:42,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653702444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:42,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/362580f0d5cc4010a70a96e7ca2f8f5e 2024-11-26T20:40:42,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T20:40:42,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:42,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:42,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:42,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:42,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T20:40:42,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/d6436d81097f458cb4477968631a4712 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/d6436d81097f458cb4477968631a4712 2024-11-26T20:40:42,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/d6436d81097f458cb4477968631a4712, entries=200, sequenceid=139, filesize=14.2 K 2024-11-26T20:40:42,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/cce7820acd5a4fb2b208f25079a950b5 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/cce7820acd5a4fb2b208f25079a950b5 2024-11-26T20:40:42,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/cce7820acd5a4fb2b208f25079a950b5, entries=150, sequenceid=139, filesize=11.9 K 2024-11-26T20:40:42,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/362580f0d5cc4010a70a96e7ca2f8f5e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/362580f0d5cc4010a70a96e7ca2f8f5e 2024-11-26T20:40:42,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/362580f0d5cc4010a70a96e7ca2f8f5e, entries=150, sequenceid=139, filesize=11.9 K 2024-11-26T20:40:42,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f47d720597d62d3cb60d84298295c6f3 in 600ms, sequenceid=139, compaction requested=false 2024-11-26T20:40:42,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:42,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T20:40:42,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:42,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:42,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:42,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:42,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:42,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:42,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/3ddeb9baf33446b688bbec2b3faedbd3 is 50, key is test_row_0/A:col10/1732653642094/Put/seqid=0 2024-11-26T20:40:42,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741870_1046 (size=16931) 2024-11-26T20:40:42,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T20:40:42,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:42,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:42,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:42,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:42,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:42,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653702710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:42,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653702707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:42,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:42,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653702718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:42,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653702718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:42,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:42,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653702754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:42,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:42,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:42,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653702824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:42,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653702824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:42,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:42,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:42,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653702844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:42,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:42,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653702844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:42,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:42,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:42,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:42,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T20:40:42,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:42,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:42,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:42,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:42,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:43,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T20:40:43,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:43,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:43,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:43,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:43,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:43,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:43,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:43,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653703034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:43,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:43,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653703035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:43,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:43,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:43,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:43,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653703054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:43,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:43,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653703054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T20:40:43,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/3ddeb9baf33446b688bbec2b3faedbd3 2024-11-26T20:40:43,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/28132a33635c4a798364c8bc18c4689e is 50, key is test_row_0/B:col10/1732653642094/Put/seqid=0 2024-11-26T20:40:43,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741871_1047 (size=12151) 2024-11-26T20:40:43,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T20:40:43,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:43,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:43,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:43,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:43,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:43,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:43,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653703264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:43,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:43,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:43,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:43,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T20:40:43,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:43,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:43,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:43,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:43,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:43,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:43,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:43,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653703344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:43,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:43,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653703344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:43,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:43,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:43,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:43,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:43,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653703364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653703365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:43,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:43,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:43,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:43,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:43,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:43,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:43,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T20:40:43,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:43,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:43,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:43,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:43,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:43,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:43,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/28132a33635c4a798364c8bc18c4689e 2024-11-26T20:40:43,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/2f13ac24c6a94bac8a4bf6e42ad0e7da is 50, key is test_row_0/C:col10/1732653642094/Put/seqid=0 2024-11-26T20:40:43,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741872_1048 (size=12151) 2024-11-26T20:40:43,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/2f13ac24c6a94bac8a4bf6e42ad0e7da 2024-11-26T20:40:43,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/3ddeb9baf33446b688bbec2b3faedbd3 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/3ddeb9baf33446b688bbec2b3faedbd3 2024-11-26T20:40:43,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/3ddeb9baf33446b688bbec2b3faedbd3, entries=250, sequenceid=158, filesize=16.5 K 2024-11-26T20:40:43,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/28132a33635c4a798364c8bc18c4689e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/28132a33635c4a798364c8bc18c4689e 2024-11-26T20:40:43,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/28132a33635c4a798364c8bc18c4689e, entries=150, sequenceid=158, filesize=11.9 K 2024-11-26T20:40:43,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/2f13ac24c6a94bac8a4bf6e42ad0e7da as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/2f13ac24c6a94bac8a4bf6e42ad0e7da 2024-11-26T20:40:43,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/2f13ac24c6a94bac8a4bf6e42ad0e7da, entries=150, sequenceid=158, filesize=11.9 K 2024-11-26T20:40:43,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f47d720597d62d3cb60d84298295c6f3 in 935ms, sequenceid=158, compaction requested=true 2024-11-26T20:40:43,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:43,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:40:43,615 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:43,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:43,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:40:43,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:43,615 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:43,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:40:43,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:43,617 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43781 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:43,617 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/A is initiating minor compaction (all files) 2024-11-26T20:40:43,617 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/A in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:43,617 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/e7d83af8a1c649f3a7c32a04372534b7, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/d6436d81097f458cb4477968631a4712, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/3ddeb9baf33446b688bbec2b3faedbd3] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=42.8 K 2024-11-26T20:40:43,618 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:43,618 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/B is initiating minor compaction (all files) 2024-11-26T20:40:43,618 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/B in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:43,618 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/24b5cbe4151a4f4fa05ba8089aa9d086, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/cce7820acd5a4fb2b208f25079a950b5, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/28132a33635c4a798364c8bc18c4689e] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=35.8 K 2024-11-26T20:40:43,618 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7d83af8a1c649f3a7c32a04372534b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732653640491 2024-11-26T20:40:43,620 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 24b5cbe4151a4f4fa05ba8089aa9d086, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732653640491 2024-11-26T20:40:43,620 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6436d81097f458cb4477968631a4712, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732653641285 2024-11-26T20:40:43,621 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting cce7820acd5a4fb2b208f25079a950b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732653641285 2024-11-26T20:40:43,621 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ddeb9baf33446b688bbec2b3faedbd3, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732653642094 2024-11-26T20:40:43,621 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 28132a33635c4a798364c8bc18c4689e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732653642094 2024-11-26T20:40:43,639 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:43,640 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/5e255e7decd841348b83af200f4efc24 is 50, key is test_row_0/B:col10/1732653642094/Put/seqid=0 2024-11-26T20:40:43,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T20:40:43,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:43,649 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T20:40:43,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:43,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:43,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:43,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:43,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:43,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:43,651 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#A#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:43,652 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/c4ce64092d964d17867a329d9554ebdf is 50, key is test_row_0/A:col10/1732653642094/Put/seqid=0 2024-11-26T20:40:43,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/0cfaf324f9fe457fa78113fe5c6132d3 is 50, key is test_row_0/A:col10/1732653642712/Put/seqid=0 2024-11-26T20:40:43,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741873_1049 (size=12561) 2024-11-26T20:40:43,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741874_1050 (size=12561) 2024-11-26T20:40:43,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741875_1051 (size=12151) 2024-11-26T20:40:43,684 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/0cfaf324f9fe457fa78113fe5c6132d3 2024-11-26T20:40:43,694 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/c4ce64092d964d17867a329d9554ebdf as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/c4ce64092d964d17867a329d9554ebdf 2024-11-26T20:40:43,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/da4c876fdd604f2da5e6b57bf5e5e209 is 50, key is test_row_0/B:col10/1732653642712/Put/seqid=0 2024-11-26T20:40:43,707 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/A of f47d720597d62d3cb60d84298295c6f3 into c4ce64092d964d17867a329d9554ebdf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:43,707 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:43,707 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/A, priority=13, startTime=1732653643615; duration=0sec 2024-11-26T20:40:43,708 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:43,708 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:A 2024-11-26T20:40:43,708 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:43,709 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:43,709 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/C is initiating minor compaction (all files) 2024-11-26T20:40:43,709 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/C in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:43,710 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/9e3dc9f2d29246768d1b02bfdce972fc, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/362580f0d5cc4010a70a96e7ca2f8f5e, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/2f13ac24c6a94bac8a4bf6e42ad0e7da] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=35.8 K 2024-11-26T20:40:43,710 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e3dc9f2d29246768d1b02bfdce972fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732653640491 2024-11-26T20:40:43,711 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 362580f0d5cc4010a70a96e7ca2f8f5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732653641285 2024-11-26T20:40:43,711 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f13ac24c6a94bac8a4bf6e42ad0e7da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732653642094 2024-11-26T20:40:43,732 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#C#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:43,733 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/6a933192b3524e7c918642c62dc5a109 is 50, key is test_row_0/C:col10/1732653642094/Put/seqid=0 2024-11-26T20:40:43,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741876_1052 (size=12151) 2024-11-26T20:40:43,749 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/da4c876fdd604f2da5e6b57bf5e5e209 2024-11-26T20:40:43,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/c0b12fab54e84ee3a2f04bc2c3f714d8 is 50, key is test_row_0/C:col10/1732653642712/Put/seqid=0 2024-11-26T20:40:43,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741877_1053 (size=12561) 2024-11-26T20:40:43,793 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/6a933192b3524e7c918642c62dc5a109 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/6a933192b3524e7c918642c62dc5a109 2024-11-26T20:40:43,803 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/C of f47d720597d62d3cb60d84298295c6f3 into 6a933192b3524e7c918642c62dc5a109(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:43,803 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:43,803 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/C, priority=13, startTime=1732653643615; duration=0sec 2024-11-26T20:40:43,803 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:43,803 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:C 2024-11-26T20:40:43,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741878_1054 (size=12151) 2024-11-26T20:40:43,812 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/c0b12fab54e84ee3a2f04bc2c3f714d8 2024-11-26T20:40:43,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/0cfaf324f9fe457fa78113fe5c6132d3 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/0cfaf324f9fe457fa78113fe5c6132d3 2024-11-26T20:40:43,830 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/0cfaf324f9fe457fa78113fe5c6132d3, entries=150, sequenceid=173, filesize=11.9 K 2024-11-26T20:40:43,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/da4c876fdd604f2da5e6b57bf5e5e209 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/da4c876fdd604f2da5e6b57bf5e5e209 2024-11-26T20:40:43,840 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/da4c876fdd604f2da5e6b57bf5e5e209, entries=150, sequenceid=173, filesize=11.9 K 2024-11-26T20:40:43,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/c0b12fab54e84ee3a2f04bc2c3f714d8 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/c0b12fab54e84ee3a2f04bc2c3f714d8 2024-11-26T20:40:43,854 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/c0b12fab54e84ee3a2f04bc2c3f714d8, entries=150, sequenceid=173, filesize=11.9 K 2024-11-26T20:40:43,856 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for f47d720597d62d3cb60d84298295c6f3 in 207ms, sequenceid=173, compaction requested=false 2024-11-26T20:40:43,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:43,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:43,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-26T20:40:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-26T20:40:43,863 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-26T20:40:43,863 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9180 sec 2024-11-26T20:40:43,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.9280 sec 2024-11-26T20:40:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:43,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T20:40:43,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:43,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:43,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:43,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:43,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:43,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:43,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/7dcbb82a933c49ea9cac437c68e8888e is 50, key is test_row_0/A:col10/1732653643878/Put/seqid=0 2024-11-26T20:40:43,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741879_1055 (size=12151) 2024-11-26T20:40:43,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/7dcbb82a933c49ea9cac437c68e8888e 2024-11-26T20:40:43,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/d4e96f8395ef450997426008daacce16 is 50, key is test_row_0/B:col10/1732653643878/Put/seqid=0 2024-11-26T20:40:43,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653643912, max timestamp=1732653643933Num uniques 859; 2024-11-26T20:40:43,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:40:43,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653643912, max timestamp=1732653643933Num uniques 859; 2024-11-26T20:40:43,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:40:43,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732653643921, max timestamp=1732653643938Num uniques 859; 2024-11-26T20:40:43,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-26T20:40:43,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741880_1056 (size=12151) 2024-11-26T20:40:43,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/d4e96f8395ef450997426008daacce16 2024-11-26T20:40:43,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/2cca7068580042d6a65b4f05fa909390 is 50, key is test_row_0/C:col10/1732653643878/Put/seqid=0 2024-11-26T20:40:44,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653703989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:44,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653703992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:44,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653703993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:44,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653703995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:44,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741881_1057 (size=12151) 2024-11-26T20:40:44,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/2cca7068580042d6a65b4f05fa909390 2024-11-26T20:40:44,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/7dcbb82a933c49ea9cac437c68e8888e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/7dcbb82a933c49ea9cac437c68e8888e 2024-11-26T20:40:44,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/7dcbb82a933c49ea9cac437c68e8888e, entries=150, sequenceid=185, filesize=11.9 K 2024-11-26T20:40:44,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/d4e96f8395ef450997426008daacce16 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/d4e96f8395ef450997426008daacce16 2024-11-26T20:40:44,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/d4e96f8395ef450997426008daacce16, entries=150, sequenceid=185, filesize=11.9 K 2024-11-26T20:40:44,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/2cca7068580042d6a65b4f05fa909390 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/2cca7068580042d6a65b4f05fa909390 2024-11-26T20:40:44,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/2cca7068580042d6a65b4f05fa909390, entries=150, sequenceid=185, filesize=11.9 K 2024-11-26T20:40:44,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.55 KB/134705 for f47d720597d62d3cb60d84298295c6f3 in 188ms, sequenceid=185, compaction requested=true 2024-11-26T20:40:44,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:44,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:A, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:40:44,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:44,068 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:44,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:40:44,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:44,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:40:44,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T20:40:44,070 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:44,070 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/A is initiating minor compaction (all files) 2024-11-26T20:40:44,070 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/A in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:44,070 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/c4ce64092d964d17867a329d9554ebdf, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/0cfaf324f9fe457fa78113fe5c6132d3, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/7dcbb82a933c49ea9cac437c68e8888e] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=36.0 K 2024-11-26T20:40:44,071 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4ce64092d964d17867a329d9554ebdf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732653642094 2024-11-26T20:40:44,071 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cfaf324f9fe457fa78113fe5c6132d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732653642701 2024-11-26T20:40:44,072 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dcbb82a933c49ea9cac437c68e8888e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732653643862 2024-11-26T20:40:44,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T20:40:44,084 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:40:44,085 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:40:44,087 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/5e255e7decd841348b83af200f4efc24 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/5e255e7decd841348b83af200f4efc24 2024-11-26T20:40:44,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-26T20:40:44,089 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:40:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T20:40:44,091 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:40:44,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:40:44,096 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#A#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:44,096 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/B of f47d720597d62d3cb60d84298295c6f3 into 5e255e7decd841348b83af200f4efc24(size=12.3 K), total size for store is 36.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:44,096 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/7a1a6bfce38a480caddda815a54c72f8 is 50, key is test_row_0/A:col10/1732653643878/Put/seqid=0 2024-11-26T20:40:44,097 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:44,097 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/B, priority=13, startTime=1732653643615; duration=0sec 2024-11-26T20:40:44,097 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T20:40:44,097 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:B 2024-11-26T20:40:44,097 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:B 2024-11-26T20:40:44,097 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:44,099 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:44,099 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/C is initiating minor compaction (all files) 2024-11-26T20:40:44,099 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/C in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:44,099 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/6a933192b3524e7c918642c62dc5a109, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/c0b12fab54e84ee3a2f04bc2c3f714d8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/2cca7068580042d6a65b4f05fa909390] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=36.0 K 2024-11-26T20:40:44,100 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a933192b3524e7c918642c62dc5a109, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732653642094 2024-11-26T20:40:44,101 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting c0b12fab54e84ee3a2f04bc2c3f714d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732653642701 2024-11-26T20:40:44,102 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cca7068580042d6a65b4f05fa909390, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732653643862 2024-11-26T20:40:44,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741882_1058 (size=12663) 2024-11-26T20:40:44,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:44,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=138.26 KB heapSize=394.70 KB 2024-11-26T20:40:44,125 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#C#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:44,126 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/94666524e3ee4f57b349f03fd9684930 is 50, key is test_row_0/C:col10/1732653643878/Put/seqid=0 2024-11-26T20:40:44,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732653643976, max timestamp=1732653644122Num uniques 1068; 2024-11-26T20:40:44,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-26T20:40:44,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732653643976, max timestamp=1732653644122Num uniques 1068; 2024-11-26T20:40:44,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-26T20:40:44,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:44,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:40:44,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:44,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732653643976, max timestamp=1732653644122Num uniques 1009; 2024-11-26T20:40:44,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:40:44,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:44,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-26T20:40:44,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:40:44,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741883_1059 (size=12663) 2024-11-26T20:40:44,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/bb38e15477e74762ba301d1731a69bb0 is 50, key is test_row_0/A:col10/1732653644122/Put/seqid=0 2024-11-26T20:40:44,168 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/94666524e3ee4f57b349f03fd9684930 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/94666524e3ee4f57b349f03fd9684930 2024-11-26T20:40:44,180 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/C of f47d720597d62d3cb60d84298295c6f3 into 94666524e3ee4f57b349f03fd9684930(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:44,180 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:44,180 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/C, priority=13, startTime=1732653644068; duration=0sec 2024-11-26T20:40:44,181 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:44,182 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:C 2024-11-26T20:40:44,182 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:44,184 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:44,185 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/B is initiating minor compaction (all files) 2024-11-26T20:40:44,185 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/B in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:44,186 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/5e255e7decd841348b83af200f4efc24, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/da4c876fdd604f2da5e6b57bf5e5e209, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/d4e96f8395ef450997426008daacce16] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=36.0 K 2024-11-26T20:40:44,186 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e255e7decd841348b83af200f4efc24, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732653642094 2024-11-26T20:40:44,187 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting da4c876fdd604f2da5e6b57bf5e5e209, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732653642701 2024-11-26T20:40:44,188 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting d4e96f8395ef450997426008daacce16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732653643862 2024-11-26T20:40:44,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741884_1060 (size=12151) 2024-11-26T20:40:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T20:40:44,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653644164, max timestamp=1732653644185Num uniques 859; 2024-11-26T20:40:44,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:40:44,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653644164, max timestamp=1732653644185Num uniques 859; 2024-11-26T20:40:44,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:40:44,203 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:44,204 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/387567e5227f42e08ebf0d5ee62c2eb8 is 50, key is test_row_0/B:col10/1732653643878/Put/seqid=0 2024-11-26T20:40:44,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732653644164, max timestamp=1732653644185Num uniques 859; 2024-11-26T20:40:44,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-26T20:40:44,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741885_1061 (size=12663) 2024-11-26T20:40:44,245 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/387567e5227f42e08ebf0d5ee62c2eb8 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/387567e5227f42e08ebf0d5ee62c2eb8 2024-11-26T20:40:44,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T20:40:44,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:44,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:44,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:44,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:44,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:44,258 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/B of f47d720597d62d3cb60d84298295c6f3 into 387567e5227f42e08ebf0d5ee62c2eb8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:44,258 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:44,258 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/B, priority=13, startTime=1732653644068; duration=0sec 2024-11-26T20:40:44,258 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:44,258 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:B 2024-11-26T20:40:44,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732653644224, max timestamp=1732653644254Num uniques 1018; 2024-11-26T20:40:44,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-26T20:40:44,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732653644224, max timestamp=1732653644254Num uniques 1018; 2024-11-26T20:40:44,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-26T20:40:44,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9975, totalHeapSize=22912, min timestamp=1732653644224, max timestamp=1732653644254Num uniques 1168; 2024-11-26T20:40:44,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53485, new segment data size=9975, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-11-26T20:40:44,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732653644284, max timestamp=1732653644305Num uniques 1077; 2024-11-26T20:40:44,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-26T20:40:44,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=427, cellSize=19552, totalHeapSize=44648, min timestamp=1732653644284, max timestamp=1732653644305Num uniques 1077; 2024-11-26T20:40:44,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=19552, suffix heap size=139976,new segment heap size=44648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=427 2024-11-26T20:40:44,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732653644288, max timestamp=1732653644305Num uniques 1077; 2024-11-26T20:40:44,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-26T20:40:44,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:44,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:44,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653704340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:44,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653704339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:44,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732653644333, max timestamp=1732653644353Num uniques 1286; 2024-11-26T20:40:44,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=58896, new segment data size=8516, suffix heap size=161712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1286, new segment cells count=186 2024-11-26T20:40:44,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:44,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732653644333, max timestamp=1732653644353Num uniques 1136; 2024-11-26T20:40:44,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8509, totalHeapSize=19584, min timestamp=1732653644333, max timestamp=1732653644355Num uniques 1036; 2024-11-26T20:40:44,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-26T20:40:44,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47439, new segment data size=8509, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-26T20:40:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T20:40:44,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T20:40:44,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:44,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:44,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:44,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:44,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:44,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1732653644391, max timestamp=1732653644414Num uniques 1045; 2024-11-26T20:40:44,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=15800, suffix heap size=136648,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=345 2024-11-26T20:40:44,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1732653644391, max timestamp=1732653644414Num uniques 1045; 2024-11-26T20:40:44,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=15800, suffix heap size=136648,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=345 2024-11-26T20:40:44,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1732653644391, max timestamp=1732653644414Num uniques 1045; 2024-11-26T20:40:44,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=15800, suffix heap size=136648,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=345 2024-11-26T20:40:44,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:44,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653704464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653704464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:44,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:44,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653704467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:44,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:44,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:44,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:44,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653704471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:44,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:44,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653704473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:44,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:44,529 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/7a1a6bfce38a480caddda815a54c72f8 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/7a1a6bfce38a480caddda815a54c72f8 2024-11-26T20:40:44,548 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/A of f47d720597d62d3cb60d84298295c6f3 into 7a1a6bfce38a480caddda815a54c72f8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:44,548 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:44,548 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/A, priority=13, startTime=1732653644068; duration=0sec 2024-11-26T20:40:44,549 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:44,549 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:A 2024-11-26T20:40:44,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T20:40:44,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:44,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:44,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:44,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:44,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:44,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653704584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:44,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653704585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653704585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:44,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:44,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/bb38e15477e74762ba301d1731a69bb0 2024-11-26T20:40:44,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/0c5d227630e0446cb097de369f80375f is 50, key is test_row_0/B:col10/1732653644122/Put/seqid=0 2024-11-26T20:40:44,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741886_1062 (size=12151) 2024-11-26T20:40:44,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/0c5d227630e0446cb097de369f80375f 2024-11-26T20:40:44,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/e1d97858211445bcad600b52ce1a9ae3 is 50, key is test_row_0/C:col10/1732653644122/Put/seqid=0 2024-11-26T20:40:44,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741887_1063 (size=12151) 2024-11-26T20:40:44,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.15 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/e1d97858211445bcad600b52ce1a9ae3 2024-11-26T20:40:44,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/bb38e15477e74762ba301d1731a69bb0 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/bb38e15477e74762ba301d1731a69bb0 2024-11-26T20:40:44,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/bb38e15477e74762ba301d1731a69bb0, entries=150, sequenceid=224, filesize=11.9 K 2024-11-26T20:40:44,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/0c5d227630e0446cb097de369f80375f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/0c5d227630e0446cb097de369f80375f 2024-11-26T20:40:44,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653704684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:44,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:44,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/0c5d227630e0446cb097de369f80375f, entries=150, sequenceid=224, filesize=11.9 K 2024-11-26T20:40:44,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653704684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:44,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:44,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/e1d97858211445bcad600b52ce1a9ae3 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/e1d97858211445bcad600b52ce1a9ae3 2024-11-26T20:40:44,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/e1d97858211445bcad600b52ce1a9ae3, entries=150, sequenceid=224, filesize=11.9 K 2024-11-26T20:40:44,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~28.27 KB/28952, heapSize ~68.59 KB/70240, currentSize=161.02 KB/164880 for f47d720597d62d3cb60d84298295c6f3 in 582ms, sequenceid=224, compaction requested=false 2024-11-26T20:40:44,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:44,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T20:40:44,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T20:40:44,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:44,715 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=161.02 KB heapSize=447.23 KB 2024-11-26T20:40:44,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:44,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:40:44,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:44,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:40:44,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:44,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:40:44,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/d2a151f4cf7c4c74a9f13d31bf5a3726 is 50, key is test_row_0/A:col10/1732653644470/Put/seqid=0 2024-11-26T20:40:44,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741888_1064 (size=12301) 2024-11-26T20:40:44,751 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/d2a151f4cf7c4c74a9f13d31bf5a3726 2024-11-26T20:40:44,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/22aabf17fb3d4e8896a2e9f01167dfda is 50, key is test_row_0/B:col10/1732653644470/Put/seqid=0 2024-11-26T20:40:44,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741889_1065 (size=12301) 2024-11-26T20:40:44,783 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/22aabf17fb3d4e8896a2e9f01167dfda 2024-11-26T20:40:44,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/d813761b20334e5fb3086c73dec4739a is 50, key is test_row_0/C:col10/1732653644470/Put/seqid=0 2024-11-26T20:40:44,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:44,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:44,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:44,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653704807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:44,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:44,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653704807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:44,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653704808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:44,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741890_1066 (size=12301) 2024-11-26T20:40:44,819 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/d813761b20334e5fb3086c73dec4739a 2024-11-26T20:40:44,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/d2a151f4cf7c4c74a9f13d31bf5a3726 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/d2a151f4cf7c4c74a9f13d31bf5a3726 2024-11-26T20:40:44,840 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/d2a151f4cf7c4c74a9f13d31bf5a3726, entries=150, sequenceid=333, filesize=12.0 K 2024-11-26T20:40:44,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/22aabf17fb3d4e8896a2e9f01167dfda as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/22aabf17fb3d4e8896a2e9f01167dfda 2024-11-26T20:40:44,862 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/22aabf17fb3d4e8896a2e9f01167dfda, entries=150, sequenceid=333, filesize=12.0 K 2024-11-26T20:40:44,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/d813761b20334e5fb3086c73dec4739a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/d813761b20334e5fb3086c73dec4739a 2024-11-26T20:40:44,873 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/d813761b20334e5fb3086c73dec4739a, entries=150, sequenceid=333, filesize=12.0 K 2024-11-26T20:40:44,875 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~447.19 KB/457920, currentSize=26.84 KB/27480 for f47d720597d62d3cb60d84298295c6f3 in 160ms, sequenceid=333, compaction requested=true 2024-11-26T20:40:44,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:44,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:44,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-26T20:40:44,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-26T20:40:44,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-26T20:40:44,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 786 msec 2024-11-26T20:40:44,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 795 msec 2024-11-26T20:40:44,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:44,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T20:40:44,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:44,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:44,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:44,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:44,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:44,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:44,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/f9eb5fa231a444f689b940dd457ec718 is 50, key is test_row_0/A:col10/1732653644917/Put/seqid=0 2024-11-26T20:40:44,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741891_1067 (size=12297) 2024-11-26T20:40:45,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732653644998, max timestamp=1732653645019Num uniques 859; 2024-11-26T20:40:45,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T20:40:45,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732653644998, max timestamp=1732653645019Num uniques 859; 2024-11-26T20:40:45,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T20:40:45,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732653644998, max timestamp=1732653645019Num uniques 859; 2024-11-26T20:40:45,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T20:40:45,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:45,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653705061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:45,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:45,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:45,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653705066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:45,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:45,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:45,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653705070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:45,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:45,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653705070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:45,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:45,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653705074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:45,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:45,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653705174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:45,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653705185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:45,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:45,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653705195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:45,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653705195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:45,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:45,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653705195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:45,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:45,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T20:40:45,224 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:40:45,225 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:40:45,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-26T20:40:45,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T20:40:45,228 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:40:45,229 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:40:45,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:40:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T20:40:45,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/f9eb5fa231a444f689b940dd457ec718 2024-11-26T20:40:45,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/b6705fb850934563873dd0284e993b8c is 50, key is test_row_0/B:col10/1732653644917/Put/seqid=0 2024-11-26T20:40:45,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741892_1068 (size=9857) 2024-11-26T20:40:45,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T20:40:45,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:45,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:45,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:45,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:45,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:45,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:45,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653705384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:45,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:45,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653705394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:45,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:45,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:45,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653705404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:45,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:45,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653705404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653705404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:45,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T20:40:45,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:45,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:45,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:45,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:45,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:45,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T20:40:45,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T20:40:45,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:45,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:45,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:45,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:45,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:45,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653705695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:45,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:45,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653705705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:45,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:45,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653705714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:45,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:45,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:45,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653705714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653705714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:45,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:45,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:45,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:45,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/b6705fb850934563873dd0284e993b8c 2024-11-26T20:40:45,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/49016f3596df49f5aa53ff60fcebe332 is 50, key is test_row_0/C:col10/1732653644917/Put/seqid=0 2024-11-26T20:40:45,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741893_1069 (size=9857) 2024-11-26T20:40:45,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/49016f3596df49f5aa53ff60fcebe332 2024-11-26T20:40:45,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/f9eb5fa231a444f689b940dd457ec718 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f9eb5fa231a444f689b940dd457ec718 2024-11-26T20:40:45,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f9eb5fa231a444f689b940dd457ec718, entries=150, sequenceid=343, filesize=12.0 K 2024-11-26T20:40:45,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/b6705fb850934563873dd0284e993b8c as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/b6705fb850934563873dd0284e993b8c 2024-11-26T20:40:45,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/b6705fb850934563873dd0284e993b8c, entries=100, sequenceid=343, filesize=9.6 K 2024-11-26T20:40:45,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/49016f3596df49f5aa53ff60fcebe332 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/49016f3596df49f5aa53ff60fcebe332 2024-11-26T20:40:45,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/49016f3596df49f5aa53ff60fcebe332, entries=100, sequenceid=343, filesize=9.6 K 2024-11-26T20:40:45,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for f47d720597d62d3cb60d84298295c6f3 in 890ms, sequenceid=343, compaction requested=true 2024-11-26T20:40:45,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:45,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:40:45,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:45,813 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:40:45,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:40:45,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:45,813 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:40:45,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:40:45,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:45,817 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46972 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:40:45,817 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/B is initiating minor compaction (all files) 2024-11-26T20:40:45,817 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/B in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:45,817 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/387567e5227f42e08ebf0d5ee62c2eb8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/0c5d227630e0446cb097de369f80375f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/22aabf17fb3d4e8896a2e9f01167dfda, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/b6705fb850934563873dd0284e993b8c] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=45.9 K 2024-11-26T20:40:45,819 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49412 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:40:45,819 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/A is initiating minor compaction (all files) 2024-11-26T20:40:45,819 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/A in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:45,819 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/7a1a6bfce38a480caddda815a54c72f8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/bb38e15477e74762ba301d1731a69bb0, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/d2a151f4cf7c4c74a9f13d31bf5a3726, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f9eb5fa231a444f689b940dd457ec718] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=48.3 K 2024-11-26T20:40:45,820 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a1a6bfce38a480caddda815a54c72f8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732653643862 2024-11-26T20:40:45,820 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 387567e5227f42e08ebf0d5ee62c2eb8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732653643862 2024-11-26T20:40:45,821 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb38e15477e74762ba301d1731a69bb0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732653643976 2024-11-26T20:40:45,821 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c5d227630e0446cb097de369f80375f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732653643976 2024-11-26T20:40:45,822 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2a151f4cf7c4c74a9f13d31bf5a3726, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732653644460 2024-11-26T20:40:45,822 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9eb5fa231a444f689b940dd457ec718, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732653644806 2024-11-26T20:40:45,822 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 22aabf17fb3d4e8896a2e9f01167dfda, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732653644460 2024-11-26T20:40:45,823 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting b6705fb850934563873dd0284e993b8c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732653644917 2024-11-26T20:40:45,846 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:45,847 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/9ee0bae0bb3c46fc8481a6bca6a0679e is 50, key is test_row_0/A:col10/1732653644917/Put/seqid=0 2024-11-26T20:40:45,849 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#B#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:45,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T20:40:45,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:45,853 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-26T20:40:45,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:45,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:40:45,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:45,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:40:45,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:45,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:40:45,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T20:40:45,855 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/c9293b614704450ba84b98eb66eac702 is 50, key is test_row_0/B:col10/1732653644917/Put/seqid=0 2024-11-26T20:40:45,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/047c599dfbc849cdabb3fd0444fafab3 is 50, key is test_row_0/A:col10/1732653645065/Put/seqid=0 2024-11-26T20:40:45,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741894_1070 (size=12949) 2024-11-26T20:40:45,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741895_1071 (size=12949) 2024-11-26T20:40:45,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741896_1072 (size=12301) 2024-11-26T20:40:45,894 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/047c599dfbc849cdabb3fd0444fafab3 2024-11-26T20:40:45,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/c20651746214400ba92cb5b9c372ff97 is 50, key is test_row_0/B:col10/1732653645065/Put/seqid=0 2024-11-26T20:40:45,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741897_1073 (size=12301) 2024-11-26T20:40:45,916 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/c20651746214400ba92cb5b9c372ff97 2024-11-26T20:40:45,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/11adc336118542dfaee5e50ff42a637c is 50, key is test_row_0/C:col10/1732653645065/Put/seqid=0 2024-11-26T20:40:45,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741898_1074 (size=12301) 2024-11-26T20:40:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:46,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:46,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:46,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653706225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:46,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:46,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653706226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:46,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:46,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653706228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:46,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:46,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:46,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653706230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653706232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:46,283 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/9ee0bae0bb3c46fc8481a6bca6a0679e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/9ee0bae0bb3c46fc8481a6bca6a0679e 2024-11-26T20:40:46,293 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/c9293b614704450ba84b98eb66eac702 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/c9293b614704450ba84b98eb66eac702 2024-11-26T20:40:46,294 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/A of f47d720597d62d3cb60d84298295c6f3 into 9ee0bae0bb3c46fc8481a6bca6a0679e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:46,294 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:46,294 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/A, priority=12, startTime=1732653645813; duration=0sec 2024-11-26T20:40:46,294 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:46,294 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:A 2024-11-26T20:40:46,296 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:40:46,299 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46972 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:40:46,299 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/C is initiating minor compaction (all files) 2024-11-26T20:40:46,299 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/C in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:46,299 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/94666524e3ee4f57b349f03fd9684930, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/e1d97858211445bcad600b52ce1a9ae3, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/d813761b20334e5fb3086c73dec4739a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/49016f3596df49f5aa53ff60fcebe332] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=45.9 K 2024-11-26T20:40:46,300 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94666524e3ee4f57b349f03fd9684930, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732653643862 2024-11-26T20:40:46,300 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1d97858211445bcad600b52ce1a9ae3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732653643976 2024-11-26T20:40:46,302 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/B of f47d720597d62d3cb60d84298295c6f3 into c9293b614704450ba84b98eb66eac702(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:46,304 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:46,304 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/B, priority=12, startTime=1732653645813; duration=0sec 2024-11-26T20:40:46,304 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:46,304 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:B 2024-11-26T20:40:46,305 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting d813761b20334e5fb3086c73dec4739a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732653644460 2024-11-26T20:40:46,306 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49016f3596df49f5aa53ff60fcebe332, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732653644917 2024-11-26T20:40:46,318 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#C#compaction#62 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:46,319 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/6d1b11f293e742299a4ce834efb3c9c0 is 50, key is test_row_0/C:col10/1732653644917/Put/seqid=0 2024-11-26T20:40:46,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741899_1075 (size=12949) 2024-11-26T20:40:46,346 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/11adc336118542dfaee5e50ff42a637c 2024-11-26T20:40:46,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653706345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:46,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653706345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:46,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653706345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:46,360 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/6d1b11f293e742299a4ce834efb3c9c0 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/6d1b11f293e742299a4ce834efb3c9c0 2024-11-26T20:40:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T20:40:46,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/047c599dfbc849cdabb3fd0444fafab3 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/047c599dfbc849cdabb3fd0444fafab3 2024-11-26T20:40:46,375 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/C of f47d720597d62d3cb60d84298295c6f3 into 6d1b11f293e742299a4ce834efb3c9c0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:46,376 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:46,376 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/047c599dfbc849cdabb3fd0444fafab3, entries=150, sequenceid=379, filesize=12.0 K 2024-11-26T20:40:46,376 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/C, priority=12, startTime=1732653645813; duration=0sec 2024-11-26T20:40:46,377 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:46,377 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:C 2024-11-26T20:40:46,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/c20651746214400ba92cb5b9c372ff97 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/c20651746214400ba92cb5b9c372ff97 2024-11-26T20:40:46,384 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/c20651746214400ba92cb5b9c372ff97, entries=150, sequenceid=379, filesize=12.0 K 2024-11-26T20:40:46,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/11adc336118542dfaee5e50ff42a637c as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/11adc336118542dfaee5e50ff42a637c 2024-11-26T20:40:46,398 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/11adc336118542dfaee5e50ff42a637c, entries=150, sequenceid=379, filesize=12.0 K 2024-11-26T20:40:46,399 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for f47d720597d62d3cb60d84298295c6f3 in 546ms, sequenceid=379, compaction requested=false 2024-11-26T20:40:46,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:46,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:46,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-26T20:40:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-26T20:40:46,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-26T20:40:46,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1720 sec 2024-11-26T20:40:46,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.1800 sec 2024-11-26T20:40:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:46,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T20:40:46,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:46,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:46,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:46,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:46,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:46,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:46,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/8e417d22f6c143ff8bf4bb0a4c41638c is 50, key is test_row_0/A:col10/1732653646556/Put/seqid=0 2024-11-26T20:40:46,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741900_1076 (size=14741) 2024-11-26T20:40:46,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732653646573, max timestamp=1732653646604Num uniques 859; 2024-11-26T20:40:46,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T20:40:46,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732653646586, max timestamp=1732653646604Num uniques 859; 2024-11-26T20:40:46,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:40:46,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732653646586, max timestamp=1732653646604Num uniques 859; 2024-11-26T20:40:46,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:40:46,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653706676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:46,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:46,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653706678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:46,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:46,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653706680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:46,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:46,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:46,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:46,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653706783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:46,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:46,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653706794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653706794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:46,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:46,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/8e417d22f6c143ff8bf4bb0a4c41638c 2024-11-26T20:40:47,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:47,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653706994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:47,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:47,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:47,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653707004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:47,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:47,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:47,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653707004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:47,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:47,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/1f88b068eead4982a559c825ddfc19e5 is 50, key is test_row_0/B:col10/1732653646556/Put/seqid=0 2024-11-26T20:40:47,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741901_1077 (size=12301) 2024-11-26T20:40:47,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/1f88b068eead4982a559c825ddfc19e5 2024-11-26T20:40:47,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/d6b38bda5a9b42a4a8a9d397b3a4d66e is 50, key is test_row_0/C:col10/1732653646556/Put/seqid=0 2024-11-26T20:40:47,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741902_1078 (size=12301) 2024-11-26T20:40:47,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:47,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653707244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:47,250 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in f47d720597d62d3cb60d84298295c6f3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:40:47,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:47,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653707254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:47,262 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_1' in f47d720597d62d3cb60d84298295c6f3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:40:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:47,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653707314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:47,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:47,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653707314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653707314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:47,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T20:40:47,374 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:40:47,375 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:40:47,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-26T20:40:47,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T20:40:47,377 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:40:47,378 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:40:47,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:40:47,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/d6b38bda5a9b42a4a8a9d397b3a4d66e 2024-11-26T20:40:47,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/8e417d22f6c143ff8bf4bb0a4c41638c as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/8e417d22f6c143ff8bf4bb0a4c41638c 2024-11-26T20:40:47,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/8e417d22f6c143ff8bf4bb0a4c41638c, entries=200, sequenceid=394, filesize=14.4 K 2024-11-26T20:40:47,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/1f88b068eead4982a559c825ddfc19e5 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/1f88b068eead4982a559c825ddfc19e5 2024-11-26T20:40:47,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/1f88b068eead4982a559c825ddfc19e5, entries=150, sequenceid=394, filesize=12.0 K 2024-11-26T20:40:47,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/d6b38bda5a9b42a4a8a9d397b3a4d66e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/d6b38bda5a9b42a4a8a9d397b3a4d66e 2024-11-26T20:40:47,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/d6b38bda5a9b42a4a8a9d397b3a4d66e, entries=150, sequenceid=394, filesize=12.0 K 2024-11-26T20:40:47,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=123.00 KB/125950 for f47d720597d62d3cb60d84298295c6f3 in 915ms, sequenceid=394, compaction requested=true 2024-11-26T20:40:47,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:47,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:40:47,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:47,471 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:47,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:40:47,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:47,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:40:47,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:47,471 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:47,473 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:47,473 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:47,473 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/A is initiating minor compaction (all files) 2024-11-26T20:40:47,473 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/B is initiating minor compaction (all files) 2024-11-26T20:40:47,473 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/A in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:47,473 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/B in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:47,473 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/9ee0bae0bb3c46fc8481a6bca6a0679e, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/047c599dfbc849cdabb3fd0444fafab3, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/8e417d22f6c143ff8bf4bb0a4c41638c] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=39.1 K 2024-11-26T20:40:47,473 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/c9293b614704450ba84b98eb66eac702, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/c20651746214400ba92cb5b9c372ff97, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/1f88b068eead4982a559c825ddfc19e5] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=36.7 K 2024-11-26T20:40:47,474 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting c9293b614704450ba84b98eb66eac702, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732653644460 2024-11-26T20:40:47,474 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ee0bae0bb3c46fc8481a6bca6a0679e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732653644460 2024-11-26T20:40:47,474 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 047c599dfbc849cdabb3fd0444fafab3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732653645042 2024-11-26T20:40:47,474 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting c20651746214400ba92cb5b9c372ff97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732653645042 2024-11-26T20:40:47,474 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e417d22f6c143ff8bf4bb0a4c41638c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732653646227 2024-11-26T20:40:47,475 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f88b068eead4982a559c825ddfc19e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732653646554 2024-11-26T20:40:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T20:40:47,487 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#B#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:47,487 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#A#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:47,488 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/6ce15f09a12f408eab5f23e5a949cbdf is 50, key is test_row_0/B:col10/1732653646556/Put/seqid=0 2024-11-26T20:40:47,488 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/e6bf7b1c84554044b0f4100117974770 is 50, key is test_row_0/A:col10/1732653646556/Put/seqid=0 2024-11-26T20:40:47,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741904_1080 (size=13051) 2024-11-26T20:40:47,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741903_1079 (size=13051) 2024-11-26T20:40:47,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T20:40:47,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:47,531 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=123.00 KB heapSize=345.53 KB 2024-11-26T20:40:47,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:47,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:40:47,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:47,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:40:47,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:47,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:40:47,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/44ca703211494030955cb3ba88346508 is 50, key is test_row_0/A:col10/1732653646678/Put/seqid=0 2024-11-26T20:40:47,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741905_1081 (size=12301) 2024-11-26T20:40:47,547 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/44ca703211494030955cb3ba88346508 2024-11-26T20:40:47,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/b82bacfaa64a4e0981f8f6de59d6c523 is 50, key is test_row_0/B:col10/1732653646678/Put/seqid=0 2024-11-26T20:40:47,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741906_1082 (size=12301) 2024-11-26T20:40:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T20:40:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:47,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:47,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653707839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653707847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:47,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653707848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:47,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:47,917 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/e6bf7b1c84554044b0f4100117974770 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/e6bf7b1c84554044b0f4100117974770 2024-11-26T20:40:47,917 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/6ce15f09a12f408eab5f23e5a949cbdf as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/6ce15f09a12f408eab5f23e5a949cbdf 2024-11-26T20:40:47,928 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/A of f47d720597d62d3cb60d84298295c6f3 into e6bf7b1c84554044b0f4100117974770(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:47,928 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/B of f47d720597d62d3cb60d84298295c6f3 into 6ce15f09a12f408eab5f23e5a949cbdf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:47,928 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:47,928 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:47,928 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/B, priority=13, startTime=1732653647471; duration=0sec 2024-11-26T20:40:47,928 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/A, priority=13, startTime=1732653647471; duration=0sec 2024-11-26T20:40:47,928 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:47,928 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:B 2024-11-26T20:40:47,928 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:47,928 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:A 2024-11-26T20:40:47,928 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:47,930 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:47,930 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/C is initiating minor compaction (all files) 2024-11-26T20:40:47,930 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/C in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:47,930 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/6d1b11f293e742299a4ce834efb3c9c0, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/11adc336118542dfaee5e50ff42a637c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/d6b38bda5a9b42a4a8a9d397b3a4d66e] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=36.7 K 2024-11-26T20:40:47,931 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d1b11f293e742299a4ce834efb3c9c0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732653644460 2024-11-26T20:40:47,931 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 11adc336118542dfaee5e50ff42a637c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732653645042 2024-11-26T20:40:47,932 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting d6b38bda5a9b42a4a8a9d397b3a4d66e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732653646554 2024-11-26T20:40:47,941 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#C#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:47,942 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/559715dbf52b48c3bc04c307f1a95aba is 50, key is test_row_0/C:col10/1732653646556/Put/seqid=0 2024-11-26T20:40:47,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741907_1083 (size=13051) 2024-11-26T20:40:47,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:47,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653707954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653707954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:47,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653707954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:47,962 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/b82bacfaa64a4e0981f8f6de59d6c523 2024-11-26T20:40:47,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/2f4db612d6564e5092f45e737ebdc7fc is 50, key is test_row_0/C:col10/1732653646678/Put/seqid=0 2024-11-26T20:40:47,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741908_1084 (size=12301) 2024-11-26T20:40:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T20:40:48,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653708164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653708164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:48,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653708173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:48,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:48,359 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/559715dbf52b48c3bc04c307f1a95aba as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/559715dbf52b48c3bc04c307f1a95aba 2024-11-26T20:40:48,365 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/C of f47d720597d62d3cb60d84298295c6f3 into 559715dbf52b48c3bc04c307f1a95aba(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:48,365 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:48,365 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/C, priority=13, startTime=1732653647471; duration=0sec 2024-11-26T20:40:48,365 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:48,365 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:C 2024-11-26T20:40:48,381 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/2f4db612d6564e5092f45e737ebdc7fc 2024-11-26T20:40:48,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/44ca703211494030955cb3ba88346508 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/44ca703211494030955cb3ba88346508 2024-11-26T20:40:48,392 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/44ca703211494030955cb3ba88346508, entries=150, sequenceid=428, filesize=12.0 K 2024-11-26T20:40:48,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/b82bacfaa64a4e0981f8f6de59d6c523 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/b82bacfaa64a4e0981f8f6de59d6c523 2024-11-26T20:40:48,399 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/b82bacfaa64a4e0981f8f6de59d6c523, entries=150, sequenceid=428, filesize=12.0 K 2024-11-26T20:40:48,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/2f4db612d6564e5092f45e737ebdc7fc as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/2f4db612d6564e5092f45e737ebdc7fc 2024-11-26T20:40:48,407 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/2f4db612d6564e5092f45e737ebdc7fc, entries=150, sequenceid=428, filesize=12.0 K 2024-11-26T20:40:48,408 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~123.00 KB/125950, heapSize ~345.48 KB/353776, currentSize=60.38 KB/61830 for f47d720597d62d3cb60d84298295c6f3 in 877ms, sequenceid=428, compaction requested=false 2024-11-26T20:40:48,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:48,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:48,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-26T20:40:48,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-26T20:40:48,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-26T20:40:48,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0320 sec 2024-11-26T20:40:48,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.0390 sec 2024-11-26T20:40:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:48,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T20:40:48,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:48,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:48,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:48,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:48,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:48,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:48,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/30ca14fc312b418cba15e3db57692e52 is 50, key is test_row_0/A:col10/1732653648475/Put/seqid=0 2024-11-26T20:40:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T20:40:48,515 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:40:48,515 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:40:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-26T20:40:48,518 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:40:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T20:40:48,519 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:40:48,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:40:48,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741909_1085 (size=17181) 2024-11-26T20:40:48,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/30ca14fc312b418cba15e3db57692e52 2024-11-26T20:40:48,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653708521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:48,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:48,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653708527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:48,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:48,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653708530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:48,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:48,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/fa7e790088374400b8d0bd0f979d3a3e is 50, key is test_row_0/B:col10/1732653648475/Put/seqid=0 2024-11-26T20:40:48,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741910_1086 (size=12301) 2024-11-26T20:40:48,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/fa7e790088374400b8d0bd0f979d3a3e 2024-11-26T20:40:48,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/919dce535d494e15a8b7b228ebf8b2b2 is 50, key is test_row_0/C:col10/1732653648475/Put/seqid=0 2024-11-26T20:40:48,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741911_1087 (size=12301) 2024-11-26T20:40:48,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T20:40:48,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:48,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653708634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:48,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:48,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:48,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653708634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:48,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:48,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:48,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653708644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:48,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T20:40:48,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:48,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:48,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:48,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:48,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:48,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T20:40:48,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:48,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:48,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:48,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:48,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:48,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:48,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T20:40:48,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:48,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653708844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:48,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:48,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653708844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:48,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:48,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:48,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:48,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653708854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:48,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:48,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:48,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T20:40:48,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:48,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:48,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:48,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:48,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:48,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:48,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/919dce535d494e15a8b7b228ebf8b2b2 2024-11-26T20:40:48,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/30ca14fc312b418cba15e3db57692e52 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/30ca14fc312b418cba15e3db57692e52 2024-11-26T20:40:48,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/30ca14fc312b418cba15e3db57692e52, entries=250, sequenceid=445, filesize=16.8 K 2024-11-26T20:40:48,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/fa7e790088374400b8d0bd0f979d3a3e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/fa7e790088374400b8d0bd0f979d3a3e 2024-11-26T20:40:49,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/fa7e790088374400b8d0bd0f979d3a3e, entries=150, sequenceid=445, filesize=12.0 K 2024-11-26T20:40:49,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/919dce535d494e15a8b7b228ebf8b2b2 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/919dce535d494e15a8b7b228ebf8b2b2 2024-11-26T20:40:49,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/919dce535d494e15a8b7b228ebf8b2b2, entries=150, sequenceid=445, filesize=12.0 K 2024-11-26T20:40:49,011 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f47d720597d62d3cb60d84298295c6f3 in 535ms, sequenceid=445, compaction requested=true 2024-11-26T20:40:49,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:49,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:40:49,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:49,012 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:49,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:40:49,012 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:49,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:49,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:40:49,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:49,013 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:49,013 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/A is initiating minor compaction (all files) 2024-11-26T20:40:49,014 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/A in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:49,014 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/e6bf7b1c84554044b0f4100117974770, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/44ca703211494030955cb3ba88346508, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/30ca14fc312b418cba15e3db57692e52] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=41.5 K 2024-11-26T20:40:49,015 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:49,015 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6bf7b1c84554044b0f4100117974770, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732653646554 2024-11-26T20:40:49,015 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/B is initiating minor compaction (all files) 2024-11-26T20:40:49,015 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/B in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:49,015 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/6ce15f09a12f408eab5f23e5a949cbdf, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/b82bacfaa64a4e0981f8f6de59d6c523, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/fa7e790088374400b8d0bd0f979d3a3e] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=36.8 K 2024-11-26T20:40:49,016 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44ca703211494030955cb3ba88346508, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732653646674 2024-11-26T20:40:49,016 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ce15f09a12f408eab5f23e5a949cbdf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732653646554 2024-11-26T20:40:49,016 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30ca14fc312b418cba15e3db57692e52, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732653647832 2024-11-26T20:40:49,016 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting b82bacfaa64a4e0981f8f6de59d6c523, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732653646674 2024-11-26T20:40:49,017 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting fa7e790088374400b8d0bd0f979d3a3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732653647837 2024-11-26T20:40:49,026 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#A#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:49,026 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/9d448b4705a644b3a642e5c9866775eb is 50, key is test_row_0/A:col10/1732653648475/Put/seqid=0 2024-11-26T20:40:49,026 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#B#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:49,027 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/8c01b2fe1afe45588cd25c7dd2ffe2f9 is 50, key is test_row_0/B:col10/1732653648475/Put/seqid=0 2024-11-26T20:40:49,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741912_1088 (size=13153) 2024-11-26T20:40:49,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741913_1089 (size=13153) 2024-11-26T20:40:49,050 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/8c01b2fe1afe45588cd25c7dd2ffe2f9 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/8c01b2fe1afe45588cd25c7dd2ffe2f9 2024-11-26T20:40:49,057 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/B of f47d720597d62d3cb60d84298295c6f3 into 8c01b2fe1afe45588cd25c7dd2ffe2f9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:49,057 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:49,057 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/B, priority=13, startTime=1732653649012; duration=0sec 2024-11-26T20:40:49,057 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:49,057 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:B 2024-11-26T20:40:49,058 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:49,060 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:49,060 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/C is initiating minor compaction (all files) 2024-11-26T20:40:49,060 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/C in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:49,060 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/559715dbf52b48c3bc04c307f1a95aba, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/2f4db612d6564e5092f45e737ebdc7fc, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/919dce535d494e15a8b7b228ebf8b2b2] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=36.8 K 2024-11-26T20:40:49,061 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 559715dbf52b48c3bc04c307f1a95aba, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732653646554 2024-11-26T20:40:49,061 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f4db612d6564e5092f45e737ebdc7fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732653646674 2024-11-26T20:40:49,062 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 919dce535d494e15a8b7b228ebf8b2b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732653647837 2024-11-26T20:40:49,074 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#C#compaction#77 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:49,075 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/6a768fb48a48473db1cab0c7a881c92a is 50, key is test_row_0/C:col10/1732653648475/Put/seqid=0 2024-11-26T20:40:49,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741914_1090 (size=13153) 2024-11-26T20:40:49,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T20:40:49,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:49,135 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T20:40:49,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:49,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:49,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:49,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:49,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:49,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:49,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/2e243992532c47f6af06820c3f2e96dc is 50, key is test_row_0/A:col10/1732653648527/Put/seqid=0 2024-11-26T20:40:49,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T20:40:49,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741915_1091 (size=12301) 2024-11-26T20:40:49,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:49,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:49,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:49,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653709187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:49,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:49,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653709190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:49,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:49,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:49,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:49,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653709191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:49,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:49,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653709264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:49,269 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in f47d720597d62d3cb60d84298295c6f3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:40:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:49,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:49,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653709274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:49,278 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_1' in f47d720597d62d3cb60d84298295c6f3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:40:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:49,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:49,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653709294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:49,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:49,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653709294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:49,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:49,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:49,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:49,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653709304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:49,445 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/9d448b4705a644b3a642e5c9866775eb as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/9d448b4705a644b3a642e5c9866775eb 2024-11-26T20:40:49,452 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/A of f47d720597d62d3cb60d84298295c6f3 into 9d448b4705a644b3a642e5c9866775eb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:49,452 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:49,452 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/A, priority=13, startTime=1732653649012; duration=0sec 2024-11-26T20:40:49,452 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:49,452 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:A 2024-11-26T20:40:49,488 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/6a768fb48a48473db1cab0c7a881c92a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/6a768fb48a48473db1cab0c7a881c92a 2024-11-26T20:40:49,494 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/C of f47d720597d62d3cb60d84298295c6f3 into 6a768fb48a48473db1cab0c7a881c92a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:49,494 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:49,494 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/C, priority=13, startTime=1732653649012; duration=0sec 2024-11-26T20:40:49,494 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:49,494 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:C 2024-11-26T20:40:49,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:49,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653709504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:49,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:49,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653709504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:49,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:49,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653709514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:49,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:49,548 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/2e243992532c47f6af06820c3f2e96dc 2024-11-26T20:40:49,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/5679d7756eed40eea04895f3a09c210d is 50, key is test_row_0/B:col10/1732653648527/Put/seqid=0 2024-11-26T20:40:49,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741916_1092 (size=12301) 2024-11-26T20:40:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T20:40:49,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653709814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:49,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653709814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:49,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:49,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:49,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653709823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:49,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:49,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:49,962 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/5679d7756eed40eea04895f3a09c210d 2024-11-26T20:40:49,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/be958d88955a4164b98db8cb5a8fddee is 50, key is test_row_0/C:col10/1732653648527/Put/seqid=0 2024-11-26T20:40:49,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741917_1093 (size=12301) 2024-11-26T20:40:50,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:50,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653710324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:50,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:50,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653710324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:50,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:50,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:50,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:50,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:50,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:50,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:50,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:50,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653710334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:50,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:50,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:50,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:50,377 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/be958d88955a4164b98db8cb5a8fddee 2024-11-26T20:40:50,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/2e243992532c47f6af06820c3f2e96dc as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/2e243992532c47f6af06820c3f2e96dc 2024-11-26T20:40:50,389 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/2e243992532c47f6af06820c3f2e96dc, entries=150, sequenceid=464, filesize=12.0 K 2024-11-26T20:40:50,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/5679d7756eed40eea04895f3a09c210d as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/5679d7756eed40eea04895f3a09c210d 2024-11-26T20:40:50,396 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/5679d7756eed40eea04895f3a09c210d, entries=150, sequenceid=464, filesize=12.0 K 2024-11-26T20:40:50,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/be958d88955a4164b98db8cb5a8fddee as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/be958d88955a4164b98db8cb5a8fddee 2024-11-26T20:40:50,403 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/be958d88955a4164b98db8cb5a8fddee, entries=150, sequenceid=464, filesize=12.0 K 2024-11-26T20:40:50,405 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f47d720597d62d3cb60d84298295c6f3 in 1269ms, sequenceid=464, compaction requested=false 2024-11-26T20:40:50,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:50,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:50,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-26T20:40:50,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-26T20:40:50,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-26T20:40:50,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8870 sec 2024-11-26T20:40:50,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.8950 sec 2024-11-26T20:40:50,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T20:40:50,665 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:40:50,666 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:40:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-26T20:40:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T20:40:50,668 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:40:50,669 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:40:50,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:40:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T20:40:50,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T20:40:50,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:50,824 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T20:40:50,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:50,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:50,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:50,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:50,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:50,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:50,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/c940e005e4ae4306a9a474e1f1aba6dd is 50, key is test_row_0/A:col10/1732653649188/Put/seqid=0 2024-11-26T20:40:50,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741918_1094 (size=12301) 2024-11-26T20:40:50,845 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/c940e005e4ae4306a9a474e1f1aba6dd 2024-11-26T20:40:50,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/0486068531424ebe8526b201f7b612a8 is 50, key is test_row_0/B:col10/1732653649188/Put/seqid=0 2024-11-26T20:40:50,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741919_1095 (size=12301) 2024-11-26T20:40:50,870 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/0486068531424ebe8526b201f7b612a8 2024-11-26T20:40:50,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/ebd45015343a4dd7847fb03f78208e69 is 50, key is test_row_0/C:col10/1732653649188/Put/seqid=0 2024-11-26T20:40:50,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741920_1096 (size=12301) 2024-11-26T20:40:50,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T20:40:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T20:40:51,306 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/ebd45015343a4dd7847fb03f78208e69 2024-11-26T20:40:51,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/c940e005e4ae4306a9a474e1f1aba6dd as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/c940e005e4ae4306a9a474e1f1aba6dd 2024-11-26T20:40:51,317 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/c940e005e4ae4306a9a474e1f1aba6dd, entries=150, sequenceid=480, filesize=12.0 K 2024-11-26T20:40:51,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/0486068531424ebe8526b201f7b612a8 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/0486068531424ebe8526b201f7b612a8 2024-11-26T20:40:51,325 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/0486068531424ebe8526b201f7b612a8, entries=150, sequenceid=480, filesize=12.0 K 2024-11-26T20:40:51,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/ebd45015343a4dd7847fb03f78208e69 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/ebd45015343a4dd7847fb03f78208e69 2024-11-26T20:40:51,331 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/ebd45015343a4dd7847fb03f78208e69, entries=150, sequenceid=480, filesize=12.0 K 2024-11-26T20:40:51,332 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for f47d720597d62d3cb60d84298295c6f3 in 509ms, sequenceid=480, compaction requested=true 2024-11-26T20:40:51,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:51,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:51,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-26T20:40:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-26T20:40:51,336 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-26T20:40:51,336 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 665 msec 2024-11-26T20:40:51,338 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 671 msec 2024-11-26T20:40:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:51,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T20:40:51,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:51,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:51,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:51,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:51,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:51,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:51,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/907e89ea5a964a0cb6811fada28733cc is 50, key is test_row_0/A:col10/1732653651348/Put/seqid=0 2024-11-26T20:40:51,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741921_1097 (size=14741) 2024-11-26T20:40:51,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/907e89ea5a964a0cb6811fada28733cc 2024-11-26T20:40:51,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/d49101a8921f4158a69b854e7fd0d1bd is 50, key is test_row_0/B:col10/1732653651348/Put/seqid=0 2024-11-26T20:40:51,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741922_1098 (size=12301) 2024-11-26T20:40:51,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/d49101a8921f4158a69b854e7fd0d1bd 2024-11-26T20:40:51,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/f1e3399c4bd44247ae636204d3cab73b is 50, key is test_row_0/C:col10/1732653651348/Put/seqid=0 2024-11-26T20:40:51,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653651372, max timestamp=1732653651409Num uniques 859; 2024-11-26T20:40:51,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:40:51,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653651372, max timestamp=1732653651409Num uniques 859; 2024-11-26T20:40:51,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653651372, max timestamp=1732653651409Num uniques 859; 2024-11-26T20:40:51,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:40:51,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:40:51,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741923_1099 (size=12301) 2024-11-26T20:40:51,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653711455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:51,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:51,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653711457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:51,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:51,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:51,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:51,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653711458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:51,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:51,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:51,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:51,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:51,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:51,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653711575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:51,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:51,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:51,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:51,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653711575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:51,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653711575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:51,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:51,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:51,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:51,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:51,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:51,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:51,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653711785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:51,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653711785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:51,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653711786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T20:40:51,804 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:40:51,804 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:40:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-26T20:40:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T20:40:51,806 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:40:51,807 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:40:51,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:40:51,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/f1e3399c4bd44247ae636204d3cab73b 2024-11-26T20:40:51,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/907e89ea5a964a0cb6811fada28733cc as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/907e89ea5a964a0cb6811fada28733cc 2024-11-26T20:40:51,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/907e89ea5a964a0cb6811fada28733cc, entries=200, sequenceid=490, filesize=14.4 K 2024-11-26T20:40:51,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/d49101a8921f4158a69b854e7fd0d1bd as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/d49101a8921f4158a69b854e7fd0d1bd 2024-11-26T20:40:51,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/d49101a8921f4158a69b854e7fd0d1bd, entries=150, sequenceid=490, filesize=12.0 K 2024-11-26T20:40:51,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/f1e3399c4bd44247ae636204d3cab73b as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/f1e3399c4bd44247ae636204d3cab73b 2024-11-26T20:40:51,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/f1e3399c4bd44247ae636204d3cab73b, entries=150, sequenceid=490, filesize=12.0 K 2024-11-26T20:40:51,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f47d720597d62d3cb60d84298295c6f3 in 521ms, sequenceid=490, compaction requested=true 2024-11-26T20:40:51,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:51,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:40:51,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:51,873 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:40:51,873 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:40:51,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:40:51,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:51,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:40:51,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:51,875 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:40:51,875 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52496 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:40:51,875 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/B is initiating minor compaction (all files) 2024-11-26T20:40:51,875 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/A is initiating minor compaction (all files) 2024-11-26T20:40:51,875 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/B in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:51,875 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/A in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:51,875 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/8c01b2fe1afe45588cd25c7dd2ffe2f9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/5679d7756eed40eea04895f3a09c210d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/0486068531424ebe8526b201f7b612a8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/d49101a8921f4158a69b854e7fd0d1bd] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=48.9 K 2024-11-26T20:40:51,875 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/9d448b4705a644b3a642e5c9866775eb, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/2e243992532c47f6af06820c3f2e96dc, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/c940e005e4ae4306a9a474e1f1aba6dd, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/907e89ea5a964a0cb6811fada28733cc] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=51.3 K 2024-11-26T20:40:51,876 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c01b2fe1afe45588cd25c7dd2ffe2f9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732653647837 2024-11-26T20:40:51,876 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d448b4705a644b3a642e5c9866775eb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732653647837 2024-11-26T20:40:51,876 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 5679d7756eed40eea04895f3a09c210d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1732653648500 2024-11-26T20:40:51,876 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e243992532c47f6af06820c3f2e96dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1732653648500 2024-11-26T20:40:51,877 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 0486068531424ebe8526b201f7b612a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1732653649185 2024-11-26T20:40:51,879 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting d49101a8921f4158a69b854e7fd0d1bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1732653651348 2024-11-26T20:40:51,879 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting c940e005e4ae4306a9a474e1f1aba6dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1732653649185 2024-11-26T20:40:51,879 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 907e89ea5a964a0cb6811fada28733cc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1732653651347 2024-11-26T20:40:51,903 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#B#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:51,903 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#A#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:51,903 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/a5bf62ebbf444e53bfba0b78696177a5 is 50, key is test_row_0/B:col10/1732653651348/Put/seqid=0 2024-11-26T20:40:51,904 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/a2846680ec194cbfa642b6eae82823a2 is 50, key is test_row_0/A:col10/1732653651348/Put/seqid=0 2024-11-26T20:40:51,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741924_1100 (size=13289) 2024-11-26T20:40:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T20:40:51,918 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/a5bf62ebbf444e53bfba0b78696177a5 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/a5bf62ebbf444e53bfba0b78696177a5 2024-11-26T20:40:51,929 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/B of f47d720597d62d3cb60d84298295c6f3 into a5bf62ebbf444e53bfba0b78696177a5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:51,930 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:51,930 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/B, priority=12, startTime=1732653651873; duration=0sec 2024-11-26T20:40:51,930 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:51,930 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:B 2024-11-26T20:40:51,930 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:40:51,932 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:40:51,932 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/C is initiating minor compaction (all files) 2024-11-26T20:40:51,932 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/C in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:51,932 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/6a768fb48a48473db1cab0c7a881c92a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/be958d88955a4164b98db8cb5a8fddee, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/ebd45015343a4dd7847fb03f78208e69, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/f1e3399c4bd44247ae636204d3cab73b] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=48.9 K 2024-11-26T20:40:51,933 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a768fb48a48473db1cab0c7a881c92a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732653647837 2024-11-26T20:40:51,933 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting be958d88955a4164b98db8cb5a8fddee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1732653648500 2024-11-26T20:40:51,934 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting ebd45015343a4dd7847fb03f78208e69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1732653649185 2024-11-26T20:40:51,934 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting f1e3399c4bd44247ae636204d3cab73b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1732653651348 2024-11-26T20:40:51,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741925_1101 (size=13289) 2024-11-26T20:40:51,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-26T20:40:51,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:51,963 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-26T20:40:51,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:51,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:40:51,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:51,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:40:51,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:51,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:40:51,968 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#C#compaction#89 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:51,969 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/0af7a0e1c99e412ca3d6a435ebff937b is 50, key is test_row_0/C:col10/1732653651348/Put/seqid=0 2024-11-26T20:40:51,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/c7d63a8494d14766b5e4fe5e879974df is 50, key is test_row_0/A:col10/1732653651456/Put/seqid=0 2024-11-26T20:40:51,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741926_1102 (size=13289) 2024-11-26T20:40:52,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741927_1103 (size=12301) 2024-11-26T20:40:52,006 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/c7d63a8494d14766b5e4fe5e879974df 2024-11-26T20:40:52,011 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/0af7a0e1c99e412ca3d6a435ebff937b as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/0af7a0e1c99e412ca3d6a435ebff937b 2024-11-26T20:40:52,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/17c01b77a0074cd982102292e27a3e1f is 50, key is test_row_0/B:col10/1732653651456/Put/seqid=0 2024-11-26T20:40:52,026 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/C of f47d720597d62d3cb60d84298295c6f3 into 0af7a0e1c99e412ca3d6a435ebff937b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:52,026 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:52,026 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/C, priority=12, startTime=1732653651873; duration=0sec 2024-11-26T20:40:52,026 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:52,026 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:C 2024-11-26T20:40:52,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741928_1104 (size=12301) 2024-11-26T20:40:52,072 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/17c01b77a0074cd982102292e27a3e1f 2024-11-26T20:40:52,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/ca5aaa5d33244f10a6d2e3bf63a74298 is 50, key is test_row_0/C:col10/1732653651456/Put/seqid=0 2024-11-26T20:40:52,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:52,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T20:40:52,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653712109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:52,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653712123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:52,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653712124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:52,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:52,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741929_1105 (size=12301) 2024-11-26T20:40:52,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653712234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:52,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:52,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653712235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:52,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:52,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:52,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653712235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:52,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:52,360 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/a2846680ec194cbfa642b6eae82823a2 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/a2846680ec194cbfa642b6eae82823a2 2024-11-26T20:40:52,371 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/A of f47d720597d62d3cb60d84298295c6f3 into a2846680ec194cbfa642b6eae82823a2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:52,371 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:52,371 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/A, priority=12, startTime=1732653651873; duration=0sec 2024-11-26T20:40:52,371 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:52,371 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:A 2024-11-26T20:40:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T20:40:52,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653712454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:52,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:52,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653712455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:52,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:52,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653712455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:52,544 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/ca5aaa5d33244f10a6d2e3bf63a74298 2024-11-26T20:40:52,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/c7d63a8494d14766b5e4fe5e879974df as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/c7d63a8494d14766b5e4fe5e879974df 2024-11-26T20:40:52,563 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/c7d63a8494d14766b5e4fe5e879974df, entries=150, sequenceid=527, filesize=12.0 K 2024-11-26T20:40:52,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/17c01b77a0074cd982102292e27a3e1f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/17c01b77a0074cd982102292e27a3e1f 2024-11-26T20:40:52,574 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/17c01b77a0074cd982102292e27a3e1f, entries=150, sequenceid=527, filesize=12.0 K 2024-11-26T20:40:52,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/ca5aaa5d33244f10a6d2e3bf63a74298 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/ca5aaa5d33244f10a6d2e3bf63a74298 2024-11-26T20:40:52,586 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/ca5aaa5d33244f10a6d2e3bf63a74298, entries=150, sequenceid=527, filesize=12.0 K 2024-11-26T20:40:52,587 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for f47d720597d62d3cb60d84298295c6f3 in 624ms, sequenceid=527, compaction requested=false 2024-11-26T20:40:52,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:52,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:52,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-26T20:40:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-26T20:40:52,592 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-26T20:40:52,593 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 782 msec 2024-11-26T20:40:52,596 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 791 msec 2024-11-26T20:40:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:52,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T20:40:52,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:52,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:52,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:52,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:52,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:52,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:52,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/9f8db60f0565406e8927349f98b24b2b is 50, key is test_row_0/A:col10/1732653652105/Put/seqid=0 2024-11-26T20:40:52,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741930_1106 (size=17181) 2024-11-26T20:40:52,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653712833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:52,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653712835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:52,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:52,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:52,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:52,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653712840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:52,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T20:40:52,945 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:40:52,945 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:40:52,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:52,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653712944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-26T20:40:52,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:52,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653712945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-26T20:40:52,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:52,952 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:40:52,953 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:40:52,953 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:40:52,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:52,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653712954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:52,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:52,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:53,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-26T20:40:53,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-26T20:40:53,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:53,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:53,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:53,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:53,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:53,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:53,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653713154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:53,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:53,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653713154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:53,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:53,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:53,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:53,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653713174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:53,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:53,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/9f8db60f0565406e8927349f98b24b2b 2024-11-26T20:40:53,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/172f1a65a5c742d4a94f107b05a1c549 is 50, key is test_row_0/B:col10/1732653652105/Put/seqid=0 2024-11-26T20:40:53,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-26T20:40:53,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:53,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:53,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:53,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:53,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-26T20:40:53,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:53,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:53,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741931_1107 (size=12301) 2024-11-26T20:40:53,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/172f1a65a5c742d4a94f107b05a1c549 2024-11-26T20:40:53,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:53,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40882 deadline: 1732653713294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:53,295 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in f47d720597d62d3cb60d84298295c6f3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:40:53,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:53,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/fd7bd01486fb415cb555bbe2426bd503 is 50, key is test_row_0/C:col10/1732653652105/Put/seqid=0 2024-11-26T20:40:53,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40832 deadline: 1732653713304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:53,310 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_1' in f47d720597d62d3cb60d84298295c6f3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:40:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:53,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741932_1108 (size=12301) 2024-11-26T20:40:53,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/fd7bd01486fb415cb555bbe2426bd503 2024-11-26T20:40:53,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/9f8db60f0565406e8927349f98b24b2b as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/9f8db60f0565406e8927349f98b24b2b 2024-11-26T20:40:53,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/9f8db60f0565406e8927349f98b24b2b, entries=250, sequenceid=541, filesize=16.8 K 2024-11-26T20:40:53,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/172f1a65a5c742d4a94f107b05a1c549 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/172f1a65a5c742d4a94f107b05a1c549 2024-11-26T20:40:53,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/172f1a65a5c742d4a94f107b05a1c549, entries=150, sequenceid=541, filesize=12.0 K 2024-11-26T20:40:53,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/fd7bd01486fb415cb555bbe2426bd503 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/fd7bd01486fb415cb555bbe2426bd503 2024-11-26T20:40:53,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/fd7bd01486fb415cb555bbe2426bd503, entries=150, sequenceid=541, filesize=12.0 K 2024-11-26T20:40:53,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f47d720597d62d3cb60d84298295c6f3 in 620ms, sequenceid=541, compaction requested=true 2024-11-26T20:40:53,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:53,388 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:53,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:40:53,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:53,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:40:53,389 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:53,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:53,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:40:53,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:53,390 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42771 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:53,390 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/A is initiating minor compaction (all files) 2024-11-26T20:40:53,390 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/A in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:53,390 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/a2846680ec194cbfa642b6eae82823a2, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/c7d63a8494d14766b5e4fe5e879974df, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/9f8db60f0565406e8927349f98b24b2b] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=41.8 K 2024-11-26T20:40:53,391 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2846680ec194cbfa642b6eae82823a2, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1732653651348 2024-11-26T20:40:53,391 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:53,391 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/B is initiating minor compaction (all files) 2024-11-26T20:40:53,391 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7d63a8494d14766b5e4fe5e879974df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1732653651430 2024-11-26T20:40:53,391 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/B in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:53,391 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/a5bf62ebbf444e53bfba0b78696177a5, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/17c01b77a0074cd982102292e27a3e1f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/172f1a65a5c742d4a94f107b05a1c549] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=37.0 K 2024-11-26T20:40:53,391 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f8db60f0565406e8927349f98b24b2b, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732653652099 2024-11-26T20:40:53,392 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting a5bf62ebbf444e53bfba0b78696177a5, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1732653651348 2024-11-26T20:40:53,392 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 17c01b77a0074cd982102292e27a3e1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1732653651430 2024-11-26T20:40:53,393 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 172f1a65a5c742d4a94f107b05a1c549, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732653652099 2024-11-26T20:40:53,411 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#A#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:53,412 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/f76549b5885d47159f6548cdd763048f is 50, key is test_row_0/A:col10/1732653652105/Put/seqid=0 2024-11-26T20:40:53,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-26T20:40:53,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:53,417 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T20:40:53,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:53,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:53,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:53,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:53,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:53,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:53,423 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#B#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:53,423 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/665b4d455b9b4cafb59b92fde314a766 is 50, key is test_row_0/B:col10/1732653652105/Put/seqid=0 2024-11-26T20:40:53,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/f4b6b685298b4af684db7a7e43708b6a is 50, key is test_row_0/A:col10/1732653652820/Put/seqid=0 2024-11-26T20:40:53,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:53,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:53,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741933_1109 (size=13391) 2024-11-26T20:40:53,508 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/f76549b5885d47159f6548cdd763048f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f76549b5885d47159f6548cdd763048f 2024-11-26T20:40:53,515 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/A of f47d720597d62d3cb60d84298295c6f3 into f76549b5885d47159f6548cdd763048f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:53,515 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:53,516 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/A, priority=13, startTime=1732653653388; duration=0sec 2024-11-26T20:40:53,516 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:53,516 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:A 2024-11-26T20:40:53,516 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:53,517 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:53,517 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/C is initiating minor compaction (all files) 2024-11-26T20:40:53,522 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/C in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:53,522 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/0af7a0e1c99e412ca3d6a435ebff937b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/ca5aaa5d33244f10a6d2e3bf63a74298, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/fd7bd01486fb415cb555bbe2426bd503] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=37.0 K 2024-11-26T20:40:53,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741934_1110 (size=13391) 2024-11-26T20:40:53,523 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0af7a0e1c99e412ca3d6a435ebff937b, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1732653651348 2024-11-26T20:40:53,524 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca5aaa5d33244f10a6d2e3bf63a74298, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1732653651430 2024-11-26T20:40:53,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:53,525 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd7bd01486fb415cb555bbe2426bd503, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732653652099 2024-11-26T20:40:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653713511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:53,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:53,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653713512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:53,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:53,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:53,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653713514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:53,530 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/665b4d455b9b4cafb59b92fde314a766 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/665b4d455b9b4cafb59b92fde314a766 2024-11-26T20:40:53,537 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/B of f47d720597d62d3cb60d84298295c6f3 into 665b4d455b9b4cafb59b92fde314a766(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:53,538 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:53,538 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/B, priority=13, startTime=1732653653388; duration=0sec 2024-11-26T20:40:53,538 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:53,538 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:B 2024-11-26T20:40:53,557 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#C#compaction#99 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:53,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741935_1111 (size=12301) 2024-11-26T20:40:53,558 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/dd1ed13d87ee464d85c662227e6018eb is 50, key is test_row_0/C:col10/1732653652105/Put/seqid=0 2024-11-26T20:40:53,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741936_1112 (size=13391) 2024-11-26T20:40:53,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-26T20:40:53,578 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/dd1ed13d87ee464d85c662227e6018eb as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/dd1ed13d87ee464d85c662227e6018eb 2024-11-26T20:40:53,587 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/C of f47d720597d62d3cb60d84298295c6f3 into dd1ed13d87ee464d85c662227e6018eb(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:53,587 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:53,587 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/C, priority=13, startTime=1732653653389; duration=0sec 2024-11-26T20:40:53,587 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:53,587 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:C 2024-11-26T20:40:53,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:53,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653713634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:53,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:53,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:53,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653713635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:53,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:53,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653713635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:53,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653713844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:53,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653713844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:53,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:53,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653713844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:53,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:53,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:53,959 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/f4b6b685298b4af684db7a7e43708b6a 2024-11-26T20:40:53,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/bebbbf123ed54ae4be838d4702bef2b1 is 50, key is test_row_0/B:col10/1732653652820/Put/seqid=0 2024-11-26T20:40:53,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741937_1113 (size=12301) 2024-11-26T20:40:53,987 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/bebbbf123ed54ae4be838d4702bef2b1 2024-11-26T20:40:53,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/f22ee3abc9de46c5a641f830c4bef440 is 50, key is test_row_0/C:col10/1732653652820/Put/seqid=0 2024-11-26T20:40:54,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741938_1114 (size=12301) 2024-11-26T20:40:54,008 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/f22ee3abc9de46c5a641f830c4bef440 2024-11-26T20:40:54,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/f4b6b685298b4af684db7a7e43708b6a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f4b6b685298b4af684db7a7e43708b6a 2024-11-26T20:40:54,022 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f4b6b685298b4af684db7a7e43708b6a, entries=150, sequenceid=561, filesize=12.0 K 2024-11-26T20:40:54,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/bebbbf123ed54ae4be838d4702bef2b1 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/bebbbf123ed54ae4be838d4702bef2b1 2024-11-26T20:40:54,029 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/bebbbf123ed54ae4be838d4702bef2b1, entries=150, sequenceid=561, filesize=12.0 K 2024-11-26T20:40:54,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/f22ee3abc9de46c5a641f830c4bef440 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/f22ee3abc9de46c5a641f830c4bef440 2024-11-26T20:40:54,041 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/f22ee3abc9de46c5a641f830c4bef440, entries=150, sequenceid=561, filesize=12.0 K 2024-11-26T20:40:54,044 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f47d720597d62d3cb60d84298295c6f3 in 627ms, sequenceid=561, compaction requested=false 2024-11-26T20:40:54,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:54,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:54,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-26T20:40:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-26T20:40:54,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-26T20:40:54,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0930 sec 2024-11-26T20:40:54,052 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.1040 sec 2024-11-26T20:40:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-26T20:40:54,084 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:40:54,085 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:40:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-26T20:40:54,087 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:40:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-26T20:40:54,089 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:40:54,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:40:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:54,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T20:40:54,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:54,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:54,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:54,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:54,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:54,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:54,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/d7c5051f2c814d52842cc2045e0221ec is 50, key is test_row_0/A:col10/1732653654155/Put/seqid=0 2024-11-26T20:40:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-26T20:40:54,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653714197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:54,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:54,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653714205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:54,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:54,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741939_1115 (size=14741) 2024-11-26T20:40:54,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/d7c5051f2c814d52842cc2045e0221ec 2024-11-26T20:40:54,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653714207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:54,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/20e1a87b5adf43b38c39a1f49b98914f is 50, key is test_row_0/B:col10/1732653654155/Put/seqid=0 2024-11-26T20:40:54,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-26T20:40:54,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:54,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:54,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:54,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:54,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:54,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741940_1116 (size=12301) 2024-11-26T20:40:54,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653714314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:54,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653714314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:54,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653714327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:54,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:54,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-26T20:40:54,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:54,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:54,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:54,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:54,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-26T20:40:54,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653714525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:54,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653714525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:54,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653714534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:54,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:54,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-26T20:40:54,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:54,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:54,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:54,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:54,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:54,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/20e1a87b5adf43b38c39a1f49b98914f 2024-11-26T20:40:54,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/5d1e0bffd7de4529ad9ad40f87f0748f is 50, key is test_row_0/C:col10/1732653654155/Put/seqid=0 2024-11-26T20:40:54,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-26T20:40:54,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:54,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:54,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:54,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:54,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-26T20:40:54,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741941_1117 (size=12301) 2024-11-26T20:40:54,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653714834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:54,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653714844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653714844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:54,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:54,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:54,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-26T20:40:54,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:54,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:54,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:54,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:54,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:55,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-26T20:40:55,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:55,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:55,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:55,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:55,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:55,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/5d1e0bffd7de4529ad9ad40f87f0748f 2024-11-26T20:40:55,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/d7c5051f2c814d52842cc2045e0221ec as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/d7c5051f2c814d52842cc2045e0221ec 2024-11-26T20:40:55,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/d7c5051f2c814d52842cc2045e0221ec, entries=200, sequenceid=577, filesize=14.4 K 2024-11-26T20:40:55,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/20e1a87b5adf43b38c39a1f49b98914f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/20e1a87b5adf43b38c39a1f49b98914f 2024-11-26T20:40:55,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/20e1a87b5adf43b38c39a1f49b98914f, entries=150, sequenceid=577, filesize=12.0 K 2024-11-26T20:40:55,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/5d1e0bffd7de4529ad9ad40f87f0748f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/5d1e0bffd7de4529ad9ad40f87f0748f 2024-11-26T20:40:55,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-26T20:40:55,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:55,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:55,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:55,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:55,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:55,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:55,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/5d1e0bffd7de4529ad9ad40f87f0748f, entries=150, sequenceid=577, filesize=12.0 K 2024-11-26T20:40:55,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f47d720597d62d3cb60d84298295c6f3 in 1018ms, sequenceid=577, compaction requested=true 2024-11-26T20:40:55,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:55,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:40:55,174 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:55,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:55,174 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:55,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:40:55,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:55,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:40:55,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:55,175 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:55,175 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/A is initiating minor compaction (all files) 2024-11-26T20:40:55,175 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/A in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:55,175 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:55,176 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f76549b5885d47159f6548cdd763048f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f4b6b685298b4af684db7a7e43708b6a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/d7c5051f2c814d52842cc2045e0221ec] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=39.5 K 2024-11-26T20:40:55,176 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/B is initiating minor compaction (all files) 2024-11-26T20:40:55,176 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/B in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:55,176 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/665b4d455b9b4cafb59b92fde314a766, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/bebbbf123ed54ae4be838d4702bef2b1, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/20e1a87b5adf43b38c39a1f49b98914f] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=37.1 K 2024-11-26T20:40:55,176 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting f76549b5885d47159f6548cdd763048f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732653652099 2024-11-26T20:40:55,176 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4b6b685298b4af684db7a7e43708b6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=561, earliestPutTs=1732653652820 2024-11-26T20:40:55,176 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 665b4d455b9b4cafb59b92fde314a766, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732653652099 2024-11-26T20:40:55,177 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting bebbbf123ed54ae4be838d4702bef2b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=561, earliestPutTs=1732653652820 2024-11-26T20:40:55,177 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7c5051f2c814d52842cc2045e0221ec, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1732653653507 2024-11-26T20:40:55,177 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 20e1a87b5adf43b38c39a1f49b98914f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1732653653507 2024-11-26T20:40:55,194 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#A#compaction#106 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:55,194 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#B#compaction#105 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:55,195 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/1c0483ea37cd4c129935c2388af63f87 is 50, key is test_row_0/A:col10/1732653654155/Put/seqid=0 2024-11-26T20:40:55,195 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/7e37ab7c9ea04e64b4182b2e2f4e680a is 50, key is test_row_0/B:col10/1732653654155/Put/seqid=0 2024-11-26T20:40:55,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741942_1118 (size=13493) 2024-11-26T20:40:55,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741943_1119 (size=13493) 2024-11-26T20:40:55,218 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/1c0483ea37cd4c129935c2388af63f87 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/1c0483ea37cd4c129935c2388af63f87 2024-11-26T20:40:55,222 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/7e37ab7c9ea04e64b4182b2e2f4e680a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/7e37ab7c9ea04e64b4182b2e2f4e680a 2024-11-26T20:40:55,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-26T20:40:55,225 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/A of f47d720597d62d3cb60d84298295c6f3 into 1c0483ea37cd4c129935c2388af63f87(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:55,226 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:55,226 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/A, priority=13, startTime=1732653655174; duration=0sec 2024-11-26T20:40:55,226 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:55,226 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:A 2024-11-26T20:40:55,226 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:55,227 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:55,227 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/C is initiating minor compaction (all files) 2024-11-26T20:40:55,227 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/C in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:55,227 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/dd1ed13d87ee464d85c662227e6018eb, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/f22ee3abc9de46c5a641f830c4bef440, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/5d1e0bffd7de4529ad9ad40f87f0748f] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=37.1 K 2024-11-26T20:40:55,228 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd1ed13d87ee464d85c662227e6018eb, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732653652099 2024-11-26T20:40:55,230 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting f22ee3abc9de46c5a641f830c4bef440, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=561, earliestPutTs=1732653652820 2024-11-26T20:40:55,230 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/B of f47d720597d62d3cb60d84298295c6f3 into 7e37ab7c9ea04e64b4182b2e2f4e680a(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:55,230 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d1e0bffd7de4529ad9ad40f87f0748f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1732653653507 2024-11-26T20:40:55,230 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:55,230 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/B, priority=13, startTime=1732653655174; duration=0sec 2024-11-26T20:40:55,231 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:55,231 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:B 2024-11-26T20:40:55,247 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#C#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:55,248 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/37d08d0354c14b0b84396384a38cb1f5 is 50, key is test_row_0/C:col10/1732653654155/Put/seqid=0 2024-11-26T20:40:55,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741944_1120 (size=13493) 2024-11-26T20:40:55,267 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/37d08d0354c14b0b84396384a38cb1f5 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/37d08d0354c14b0b84396384a38cb1f5 2024-11-26T20:40:55,273 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/C of f47d720597d62d3cb60d84298295c6f3 into 37d08d0354c14b0b84396384a38cb1f5(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:55,273 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:55,273 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/C, priority=13, startTime=1732653655174; duration=0sec 2024-11-26T20:40:55,273 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:55,273 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:C 2024-11-26T20:40:55,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-26T20:40:55,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:55,322 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T20:40:55,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:55,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:55,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:55,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:55,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:55,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:55,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/1173ae76ff164d98bebddeb63e613d10 is 50, key is test_row_0/A:col10/1732653654195/Put/seqid=0 2024-11-26T20:40:55,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741945_1121 (size=12301) 2024-11-26T20:40:55,366 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/1173ae76ff164d98bebddeb63e613d10 2024-11-26T20:40:55,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/87607341c5ba4ec480aa2120536fc263 is 50, key is test_row_0/B:col10/1732653654195/Put/seqid=0 2024-11-26T20:40:55,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:55,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:55,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653715407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:55,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:55,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653715409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:55,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:55,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653715411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:55,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:55,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741946_1122 (size=12301) 2024-11-26T20:40:55,427 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/87607341c5ba4ec480aa2120536fc263 2024-11-26T20:40:55,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/85e7158991674c14a3cd98a7556c7170 is 50, key is test_row_0/C:col10/1732653654195/Put/seqid=0 2024-11-26T20:40:55,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741947_1123 (size=12301) 2024-11-26T20:40:55,476 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/85e7158991674c14a3cd98a7556c7170 2024-11-26T20:40:55,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/1173ae76ff164d98bebddeb63e613d10 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/1173ae76ff164d98bebddeb63e613d10 2024-11-26T20:40:55,489 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/1173ae76ff164d98bebddeb63e613d10, entries=150, sequenceid=599, filesize=12.0 K 2024-11-26T20:40:55,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/87607341c5ba4ec480aa2120536fc263 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/87607341c5ba4ec480aa2120536fc263 2024-11-26T20:40:55,495 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/87607341c5ba4ec480aa2120536fc263, entries=150, sequenceid=599, filesize=12.0 K 2024-11-26T20:40:55,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/85e7158991674c14a3cd98a7556c7170 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/85e7158991674c14a3cd98a7556c7170 2024-11-26T20:40:55,512 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/85e7158991674c14a3cd98a7556c7170, entries=150, sequenceid=599, filesize=12.0 K 2024-11-26T20:40:55,513 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f47d720597d62d3cb60d84298295c6f3 in 191ms, sequenceid=599, compaction requested=false 2024-11-26T20:40:55,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:55,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:55,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-26T20:40:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-26T20:40:55,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-26T20:40:55,523 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4260 sec 2024-11-26T20:40:55,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.4380 sec 2024-11-26T20:40:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:55,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T20:40:55,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:55,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:55,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:55,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:55,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:55,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:55,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/ba03503bff7a4049a57fcdab923156fe is 50, key is test_row_0/A:col10/1732653655396/Put/seqid=0 2024-11-26T20:40:55,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741948_1124 (size=12301) 2024-11-26T20:40:55,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653715569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:55,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653715571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653715573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:55,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:55,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653715683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:55,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653715684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:55,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:55,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653715684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:55,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:55,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:55,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653715894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:55,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:55,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653715905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:55,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653715905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:55,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:55,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/ba03503bff7a4049a57fcdab923156fe 2024-11-26T20:40:55,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/24eb0d05f03f4d18b10314861ca2c45a is 50, key is test_row_0/B:col10/1732653655396/Put/seqid=0 2024-11-26T20:40:56,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741949_1125 (size=12301) 2024-11-26T20:40:56,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/24eb0d05f03f4d18b10314861ca2c45a 2024-11-26T20:40:56,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/9525f41f8bb749b7aa8aecabf7694ebd is 50, key is test_row_0/C:col10/1732653655396/Put/seqid=0 2024-11-26T20:40:56,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741950_1126 (size=12301) 2024-11-26T20:40:56,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/9525f41f8bb749b7aa8aecabf7694ebd 2024-11-26T20:40:56,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/ba03503bff7a4049a57fcdab923156fe as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/ba03503bff7a4049a57fcdab923156fe 2024-11-26T20:40:56,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/ba03503bff7a4049a57fcdab923156fe, entries=150, sequenceid=614, filesize=12.0 K 2024-11-26T20:40:56,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/24eb0d05f03f4d18b10314861ca2c45a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/24eb0d05f03f4d18b10314861ca2c45a 2024-11-26T20:40:56,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/24eb0d05f03f4d18b10314861ca2c45a, entries=150, sequenceid=614, filesize=12.0 K 2024-11-26T20:40:56,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/9525f41f8bb749b7aa8aecabf7694ebd as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/9525f41f8bb749b7aa8aecabf7694ebd 2024-11-26T20:40:56,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/9525f41f8bb749b7aa8aecabf7694ebd, entries=150, sequenceid=614, filesize=12.0 K 2024-11-26T20:40:56,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f47d720597d62d3cb60d84298295c6f3 in 545ms, sequenceid=614, compaction requested=true 2024-11-26T20:40:56,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:56,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:40:56,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:56,076 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:56,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:40:56,076 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:56,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:56,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:40:56,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:56,078 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:56,078 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:56,078 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/B is initiating minor compaction (all files) 2024-11-26T20:40:56,078 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/A is initiating minor compaction (all files) 2024-11-26T20:40:56,078 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/B in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:56,078 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/A in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:56,078 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/7e37ab7c9ea04e64b4182b2e2f4e680a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/87607341c5ba4ec480aa2120536fc263, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/24eb0d05f03f4d18b10314861ca2c45a] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=37.2 K 2024-11-26T20:40:56,078 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/1c0483ea37cd4c129935c2388af63f87, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/1173ae76ff164d98bebddeb63e613d10, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/ba03503bff7a4049a57fcdab923156fe] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=37.2 K 2024-11-26T20:40:56,078 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c0483ea37cd4c129935c2388af63f87, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1732653653507 2024-11-26T20:40:56,078 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e37ab7c9ea04e64b4182b2e2f4e680a, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1732653653507 2024-11-26T20:40:56,079 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 87607341c5ba4ec480aa2120536fc263, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1732653654195 2024-11-26T20:40:56,079 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1173ae76ff164d98bebddeb63e613d10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1732653654195 2024-11-26T20:40:56,079 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 24eb0d05f03f4d18b10314861ca2c45a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1732653655396 2024-11-26T20:40:56,079 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba03503bff7a4049a57fcdab923156fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1732653655396 2024-11-26T20:40:56,093 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#A#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:56,093 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#B#compaction#115 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:56,094 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/50c4dd609c55436f9e79339d6fe90982 is 50, key is test_row_0/A:col10/1732653655396/Put/seqid=0 2024-11-26T20:40:56,094 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/d526f38ef81d4c0682c7e0867438f601 is 50, key is test_row_0/B:col10/1732653655396/Put/seqid=0 2024-11-26T20:40:56,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741951_1127 (size=13595) 2024-11-26T20:40:56,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741952_1128 (size=13595) 2024-11-26T20:40:56,153 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/d526f38ef81d4c0682c7e0867438f601 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/d526f38ef81d4c0682c7e0867438f601 2024-11-26T20:40:56,161 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/B of f47d720597d62d3cb60d84298295c6f3 into d526f38ef81d4c0682c7e0867438f601(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:56,161 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:56,161 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/B, priority=13, startTime=1732653656076; duration=0sec 2024-11-26T20:40:56,161 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:56,162 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:B 2024-11-26T20:40:56,162 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:40:56,163 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:40:56,163 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/C is initiating minor compaction (all files) 2024-11-26T20:40:56,163 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/C in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:56,163 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/37d08d0354c14b0b84396384a38cb1f5, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/85e7158991674c14a3cd98a7556c7170, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/9525f41f8bb749b7aa8aecabf7694ebd] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=37.2 K 2024-11-26T20:40:56,164 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 37d08d0354c14b0b84396384a38cb1f5, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1732653653507 2024-11-26T20:40:56,164 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 85e7158991674c14a3cd98a7556c7170, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1732653654195 2024-11-26T20:40:56,165 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 9525f41f8bb749b7aa8aecabf7694ebd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1732653655396 2024-11-26T20:40:56,187 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#C#compaction#116 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:56,187 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/c0f33cbdf5784fa5808ba6d430fbd1b3 is 50, key is test_row_0/C:col10/1732653655396/Put/seqid=0 2024-11-26T20:40:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:56,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T20:40:56,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:56,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:56,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:56,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:56,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:56,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:56,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/fa34a911f0324fa6965aea96cd1b98d2 is 50, key is test_row_0/A:col10/1732653656209/Put/seqid=0 2024-11-26T20:40:56,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741953_1129 (size=13595) 2024-11-26T20:40:56,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741954_1130 (size=17181) 2024-11-26T20:40:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-26T20:40:56,236 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:40:56,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=634 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/fa34a911f0324fa6965aea96cd1b98d2 2024-11-26T20:40:56,237 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:40:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees 2024-11-26T20:40:56,240 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:40:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-26T20:40:56,241 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:40:56,241 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:40:56,244 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/c0f33cbdf5784fa5808ba6d430fbd1b3 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/c0f33cbdf5784fa5808ba6d430fbd1b3 2024-11-26T20:40:56,254 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/C of f47d720597d62d3cb60d84298295c6f3 into c0f33cbdf5784fa5808ba6d430fbd1b3(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:56,254 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:56,254 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/C, priority=13, startTime=1732653656076; duration=0sec 2024-11-26T20:40:56,254 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:56,254 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:C 2024-11-26T20:40:56,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:56,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653716245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:56,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:56,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/4341217089ed4f598a0fd595fe48ea11 is 50, key is test_row_0/B:col10/1732653656209/Put/seqid=0 2024-11-26T20:40:56,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653716249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:56,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:56,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653716253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:56,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:56,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741955_1131 (size=12301) 2024-11-26T20:40:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-26T20:40:56,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:56,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653716364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:56,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:56,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:56,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653716364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:56,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:56,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653716364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:56,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:56,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:56,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-26T20:40:56,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:56,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:56,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:56,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:56,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:56,528 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/50c4dd609c55436f9e79339d6fe90982 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/50c4dd609c55436f9e79339d6fe90982 2024-11-26T20:40:56,537 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/A of f47d720597d62d3cb60d84298295c6f3 into 50c4dd609c55436f9e79339d6fe90982(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:56,537 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:56,537 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/A, priority=13, startTime=1732653656076; duration=0sec 2024-11-26T20:40:56,538 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:56,538 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:A 2024-11-26T20:40:56,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-26T20:40:56,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:56,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:56,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:56,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:56,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:56,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-26T20:40:56,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653716576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:56,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653716576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:56,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:56,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653716578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:56,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:56,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=634 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/4341217089ed4f598a0fd595fe48ea11 2024-11-26T20:40:56,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-26T20:40:56,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:56,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:56,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:56,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:56,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:56,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:56,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/d3cb8fd2cf04459ab967c601f28cc368 is 50, key is test_row_0/C:col10/1732653656209/Put/seqid=0 2024-11-26T20:40:56,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741956_1132 (size=12301) 2024-11-26T20:40:56,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=634 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/d3cb8fd2cf04459ab967c601f28cc368 2024-11-26T20:40:56,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/fa34a911f0324fa6965aea96cd1b98d2 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/fa34a911f0324fa6965aea96cd1b98d2 2024-11-26T20:40:56,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/fa34a911f0324fa6965aea96cd1b98d2, entries=250, sequenceid=634, filesize=16.8 K 2024-11-26T20:40:56,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/4341217089ed4f598a0fd595fe48ea11 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/4341217089ed4f598a0fd595fe48ea11 2024-11-26T20:40:56,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/4341217089ed4f598a0fd595fe48ea11, entries=150, sequenceid=634, filesize=12.0 K 2024-11-26T20:40:56,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/d3cb8fd2cf04459ab967c601f28cc368 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/d3cb8fd2cf04459ab967c601f28cc368 2024-11-26T20:40:56,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/d3cb8fd2cf04459ab967c601f28cc368, entries=150, sequenceid=634, filesize=12.0 K 2024-11-26T20:40:56,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for f47d720597d62d3cb60d84298295c6f3 in 547ms, sequenceid=634, compaction requested=false 2024-11-26T20:40:56,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:56,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-26T20:40:56,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:56,859 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T20:40:56,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:56,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:56,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:56,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:56,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:56,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-26T20:40:56,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/f19eb3372d8348a1a7193f8357171daf is 50, key is test_row_0/A:col10/1732653656218/Put/seqid=0 2024-11-26T20:40:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:56,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:56,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741957_1133 (size=12301) 2024-11-26T20:40:56,904 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/f19eb3372d8348a1a7193f8357171daf 2024-11-26T20:40:56,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/4a5fd8674e8b41c1aba642172bb9848f is 50, key is test_row_0/B:col10/1732653656218/Put/seqid=0 2024-11-26T20:40:56,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653716955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:56,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653716955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:56,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653716961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:56,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741958_1134 (size=12301) 2024-11-26T20:40:56,978 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/4a5fd8674e8b41c1aba642172bb9848f 2024-11-26T20:40:57,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/de707e004bb5441f83dd0bfae878e93a is 50, key is test_row_0/C:col10/1732653656218/Put/seqid=0 2024-11-26T20:40:57,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741959_1135 (size=12301) 2024-11-26T20:40:57,055 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/de707e004bb5441f83dd0bfae878e93a 2024-11-26T20:40:57,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653717064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:57,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/f19eb3372d8348a1a7193f8357171daf as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f19eb3372d8348a1a7193f8357171daf 2024-11-26T20:40:57,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653717074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:57,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653717074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:57,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:57,082 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f19eb3372d8348a1a7193f8357171daf, entries=150, sequenceid=650, filesize=12.0 K 2024-11-26T20:40:57,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/4a5fd8674e8b41c1aba642172bb9848f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/4a5fd8674e8b41c1aba642172bb9848f 2024-11-26T20:40:57,089 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/4a5fd8674e8b41c1aba642172bb9848f, entries=150, sequenceid=650, filesize=12.0 K 2024-11-26T20:40:57,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/de707e004bb5441f83dd0bfae878e93a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/de707e004bb5441f83dd0bfae878e93a 2024-11-26T20:40:57,100 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/de707e004bb5441f83dd0bfae878e93a, entries=150, sequenceid=650, filesize=12.0 K 2024-11-26T20:40:57,106 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f47d720597d62d3cb60d84298295c6f3 in 246ms, sequenceid=650, compaction requested=true 2024-11-26T20:40:57,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:57,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:57,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-11-26T20:40:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-11-26T20:40:57,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-26T20:40:57,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 868 msec 2024-11-26T20:40:57,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees in 875 msec 2024-11-26T20:40:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:57,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T20:40:57,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:57,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:57,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:57,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:57,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:57,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:57,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/6034b85693f9486e92438065a5d9e0b0 is 50, key is test_row_0/A:col10/1732653656923/Put/seqid=0 2024-11-26T20:40:57,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653717306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:57,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741960_1136 (size=14741) 2024-11-26T20:40:57,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653717309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:57,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=669 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/6034b85693f9486e92438065a5d9e0b0 2024-11-26T20:40:57,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:57,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653717313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:57,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:57,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/5b0f9737e92343ba9be75f61f13d148b is 50, key is test_row_0/B:col10/1732653656923/Put/seqid=0 2024-11-26T20:40:57,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741961_1137 (size=12301) 2024-11-26T20:40:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-26T20:40:57,375 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:40:57,375 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:40:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees 2024-11-26T20:40:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-26T20:40:57,378 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=31, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:40:57,378 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=31, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:40:57,379 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:40:57,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653717424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:57,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:57,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653717424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653717424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:57,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-26T20:40:57,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-26T20:40:57,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:57,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:57,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:57,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:57,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:57,631 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-26T20:40:57,631 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:40:57,634 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-26T20:40:57,634 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:40:57,637 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-26T20:40:57,637 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:40:57,638 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:57,638 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:57,638 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:57,638 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-26T20:40:57,638 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:40:57,638 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:57,638 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:57,639 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:57,639 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:40:57,639 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:40:57,639 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:57,639 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:57,639 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:40:57,641 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:40:57,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:57,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:57,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653717644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653717644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653717644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:57,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-26T20:40:57,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:57,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:57,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:57,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:57,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-26T20:40:57,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=669 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/5b0f9737e92343ba9be75f61f13d148b 2024-11-26T20:40:57,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/bf2f267646254d0da31c56f1d0477763 is 50, key is test_row_0/C:col10/1732653656923/Put/seqid=0 2024-11-26T20:40:57,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741962_1138 (size=12301) 2024-11-26T20:40:57,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-26T20:40:57,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:57,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:57,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:57,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:57,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:57,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:57,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:57,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:40:57,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40868 deadline: 1732653717954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:57,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40918 deadline: 1732653717954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:57,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40852 deadline: 1732653717954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 2024-11-26T20:40:57,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:57,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:57,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:40:57,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:40:57,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-26T20:40:57,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:57,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:57,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:57,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:57,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:57,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:58,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-26T20:40:58,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-26T20:40:58,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:58,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:58,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:58,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:58,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:58,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:40:58,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=669 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/bf2f267646254d0da31c56f1d0477763 2024-11-26T20:40:58,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/6034b85693f9486e92438065a5d9e0b0 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/6034b85693f9486e92438065a5d9e0b0 2024-11-26T20:40:58,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/6034b85693f9486e92438065a5d9e0b0, entries=200, sequenceid=669, filesize=14.4 K 2024-11-26T20:40:58,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/5b0f9737e92343ba9be75f61f13d148b as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/5b0f9737e92343ba9be75f61f13d148b 2024-11-26T20:40:58,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/5b0f9737e92343ba9be75f61f13d148b, entries=150, sequenceid=669, filesize=12.0 K 2024-11-26T20:40:58,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/bf2f267646254d0da31c56f1d0477763 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/bf2f267646254d0da31c56f1d0477763 2024-11-26T20:40:58,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/bf2f267646254d0da31c56f1d0477763, entries=150, sequenceid=669, filesize=12.0 K 2024-11-26T20:40:58,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f47d720597d62d3cb60d84298295c6f3 in 944ms, sequenceid=669, compaction requested=true 2024-11-26T20:40:58,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:58,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:40:58,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:58,221 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:40:58,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:40:58,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:58,221 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:40:58,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47d720597d62d3cb60d84298295c6f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:40:58,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:58,223 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:40:58,223 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/A is initiating minor compaction (all files) 2024-11-26T20:40:58,223 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/A in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:58,224 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/50c4dd609c55436f9e79339d6fe90982, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/fa34a911f0324fa6965aea96cd1b98d2, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f19eb3372d8348a1a7193f8357171daf, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/6034b85693f9486e92438065a5d9e0b0] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=56.5 K 2024-11-26T20:40:58,224 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50498 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:40:58,224 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 50c4dd609c55436f9e79339d6fe90982, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1732653655396 2024-11-26T20:40:58,224 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/B is initiating minor compaction (all files) 2024-11-26T20:40:58,224 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/B in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:58,224 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/d526f38ef81d4c0682c7e0867438f601, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/4341217089ed4f598a0fd595fe48ea11, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/4a5fd8674e8b41c1aba642172bb9848f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/5b0f9737e92343ba9be75f61f13d148b] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=49.3 K 2024-11-26T20:40:58,225 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting fa34a911f0324fa6965aea96cd1b98d2, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1732653655566 2024-11-26T20:40:58,225 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting d526f38ef81d4c0682c7e0867438f601, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1732653655396 2024-11-26T20:40:58,225 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting f19eb3372d8348a1a7193f8357171daf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1732653656218 2024-11-26T20:40:58,225 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4341217089ed4f598a0fd595fe48ea11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1732653655569 2024-11-26T20:40:58,226 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 6034b85693f9486e92438065a5d9e0b0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=669, earliestPutTs=1732653656923 2024-11-26T20:40:58,226 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a5fd8674e8b41c1aba642172bb9848f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1732653656218 2024-11-26T20:40:58,227 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b0f9737e92343ba9be75f61f13d148b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=669, earliestPutTs=1732653656923 2024-11-26T20:40:58,243 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#A#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:58,243 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/806989e743ef424dae12e7934589d0e7 is 50, key is test_row_0/A:col10/1732653656923/Put/seqid=0 2024-11-26T20:40:58,249 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#B#compaction#127 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:58,249 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/6fff4ae6ba2e4e0c8ef59377618d498e is 50, key is test_row_0/B:col10/1732653656923/Put/seqid=0 2024-11-26T20:40:58,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741963_1139 (size=13731) 2024-11-26T20:40:58,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741964_1140 (size=13731) 2024-11-26T20:40:58,270 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/6fff4ae6ba2e4e0c8ef59377618d498e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/6fff4ae6ba2e4e0c8ef59377618d498e 2024-11-26T20:40:58,274 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/806989e743ef424dae12e7934589d0e7 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/806989e743ef424dae12e7934589d0e7 2024-11-26T20:40:58,281 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/A of f47d720597d62d3cb60d84298295c6f3 into 806989e743ef424dae12e7934589d0e7(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:58,281 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:58,281 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/A, priority=12, startTime=1732653658221; duration=0sec 2024-11-26T20:40:58,281 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:40:58,281 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:A 2024-11-26T20:40:58,282 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:40:58,283 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50498 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:40:58,284 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): f47d720597d62d3cb60d84298295c6f3/C is initiating minor compaction (all files) 2024-11-26T20:40:58,284 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47d720597d62d3cb60d84298295c6f3/C in TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:58,284 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/c0f33cbdf5784fa5808ba6d430fbd1b3, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/d3cb8fd2cf04459ab967c601f28cc368, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/de707e004bb5441f83dd0bfae878e93a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/bf2f267646254d0da31c56f1d0477763] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp, totalSize=49.3 K 2024-11-26T20:40:58,284 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/B of f47d720597d62d3cb60d84298295c6f3 into 6fff4ae6ba2e4e0c8ef59377618d498e(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:58,284 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:58,284 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/B, priority=12, startTime=1732653658221; duration=0sec 2024-11-26T20:40:58,284 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:58,284 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting c0f33cbdf5784fa5808ba6d430fbd1b3, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1732653655396 2024-11-26T20:40:58,284 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:B 2024-11-26T20:40:58,285 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting d3cb8fd2cf04459ab967c601f28cc368, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1732653655569 2024-11-26T20:40:58,285 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting de707e004bb5441f83dd0bfae878e93a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1732653656218 2024-11-26T20:40:58,286 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting bf2f267646254d0da31c56f1d0477763, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=669, earliestPutTs=1732653656923 2024-11-26T20:40:58,298 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47d720597d62d3cb60d84298295c6f3#C#compaction#128 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:40:58,300 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/8c0bb1edd15746318fac86d3fcaeafc9 is 50, key is test_row_0/C:col10/1732653656923/Put/seqid=0 2024-11-26T20:40:58,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-26T20:40:58,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:58,304 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T20:40:58,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:40:58,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:58,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:40:58,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:58,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:40:58,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:40:58,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741965_1141 (size=13731) 2024-11-26T20:40:58,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/bdb8ea1d7835416f99e0e520ee6b882b is 50, key is test_row_0/A:col10/1732653657290/Put/seqid=0 2024-11-26T20:40:58,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741966_1142 (size=12301) 2024-11-26T20:40:58,331 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=684 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/bdb8ea1d7835416f99e0e520ee6b882b 2024-11-26T20:40:58,339 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/8c0bb1edd15746318fac86d3fcaeafc9 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/8c0bb1edd15746318fac86d3fcaeafc9 2024-11-26T20:40:58,347 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f47d720597d62d3cb60d84298295c6f3/C of f47d720597d62d3cb60d84298295c6f3 into 8c0bb1edd15746318fac86d3fcaeafc9(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:40:58,347 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:58,347 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., storeName=f47d720597d62d3cb60d84298295c6f3/C, priority=12, startTime=1732653658221; duration=0sec 2024-11-26T20:40:58,347 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:40:58,347 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47d720597d62d3cb60d84298295c6f3:C 2024-11-26T20:40:58,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/16dd20098bc940a48ad7afecda1bc558 is 50, key is test_row_0/B:col10/1732653657290/Put/seqid=0 2024-11-26T20:40:58,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741967_1143 (size=12301) 2024-11-26T20:40:58,360 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=684 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/16dd20098bc940a48ad7afecda1bc558 2024-11-26T20:40:58,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/dc97731b37bc47f59461109352b8f0ba is 50, key is test_row_0/C:col10/1732653657290/Put/seqid=0 2024-11-26T20:40:58,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741968_1144 (size=12301) 2024-11-26T20:40:58,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:40:58,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. as already flushing 2024-11-26T20:40:58,466 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-26T20:40:58,466 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-26T20:40:58,466 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-26T20:40:58,466 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:40:58,466 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:40:58,466 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:40:58,466 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:58,466 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:58,466 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:58,466 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:58,466 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:58,466 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:40:58,466 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:40:58,466 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:40:58,466 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:40:58,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-26T20:40:58,772 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=684 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/dc97731b37bc47f59461109352b8f0ba 2024-11-26T20:40:58,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/bdb8ea1d7835416f99e0e520ee6b882b as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/bdb8ea1d7835416f99e0e520ee6b882b 2024-11-26T20:40:58,781 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/bdb8ea1d7835416f99e0e520ee6b882b, entries=150, sequenceid=684, filesize=12.0 K 2024-11-26T20:40:58,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/16dd20098bc940a48ad7afecda1bc558 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/16dd20098bc940a48ad7afecda1bc558 2024-11-26T20:40:58,786 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/16dd20098bc940a48ad7afecda1bc558, entries=150, sequenceid=684, filesize=12.0 K 2024-11-26T20:40:58,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/dc97731b37bc47f59461109352b8f0ba as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/dc97731b37bc47f59461109352b8f0ba 2024-11-26T20:40:58,791 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/dc97731b37bc47f59461109352b8f0ba, entries=150, sequenceid=684, filesize=12.0 K 2024-11-26T20:40:58,792 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=20.13 KB/20610 for f47d720597d62d3cb60d84298295c6f3 in 488ms, sequenceid=684, compaction requested=false 2024-11-26T20:40:58,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2603): Flush status journal for f47d720597d62d3cb60d84298295c6f3: 2024-11-26T20:40:58,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:40:58,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=32 2024-11-26T20:40:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=32 2024-11-26T20:40:58,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-26T20:40:58,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4150 sec 2024-11-26T20:40:58,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees in 1.4190 sec 2024-11-26T20:40:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-26T20:40:59,525 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:40:59,966 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T20:41:03,386 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-26T20:41:03,386 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:41:03,386 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:03,386 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:03,386 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:41:03,395 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-26T20:41:03,395 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:41:03,395 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:03,395 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:03,395 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:41:03,395 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T20:41:03,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-26T20:41:03,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-26T20:41:03,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 129 2024-11-26T20:41:03,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 133 2024-11-26T20:41:03,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 151 2024-11-26T20:41:03,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T20:41:03,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2467 2024-11-26T20:41:03,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2393 2024-11-26T20:41:03,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T20:41:03,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1122 2024-11-26T20:41:03,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3365 rows 2024-11-26T20:41:03,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1115 2024-11-26T20:41:03,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3343 rows 2024-11-26T20:41:03,396 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T20:41:03,396 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:41:03,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:03,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:03,397 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:41:03,401 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T20:41:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T20:41:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-26T20:41:03,408 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653663408"}]},"ts":"1732653663408"} 2024-11-26T20:41:03,410 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T20:41:03,410 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T20:41:03,411 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T20:41:03,415 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f47d720597d62d3cb60d84298295c6f3, UNASSIGN}] 2024-11-26T20:41:03,416 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f47d720597d62d3cb60d84298295c6f3, UNASSIGN 2024-11-26T20:41:03,417 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=f47d720597d62d3cb60d84298295c6f3, regionState=CLOSING, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:41:03,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f47d720597d62d3cb60d84298295c6f3, UNASSIGN because future has completed 2024-11-26T20:41:03,419 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T20:41:03,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795}] 2024-11-26T20:41:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-26T20:41:03,576 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:41:03,576 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T20:41:03,576 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing f47d720597d62d3cb60d84298295c6f3, disabling compactions & flushes 2024-11-26T20:41:03,576 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:41:03,577 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:41:03,577 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. after waiting 0 ms 2024-11-26T20:41:03,577 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:41:03,577 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(2902): Flushing f47d720597d62d3cb60d84298295c6f3 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-26T20:41:03,577 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=A 2024-11-26T20:41:03,577 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:03,577 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=B 2024-11-26T20:41:03,577 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:03,577 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47d720597d62d3cb60d84298295c6f3, store=C 2024-11-26T20:41:03,577 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:03,581 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/8f2c3512527046aeb50dec2d49551a77 is 50, key is test_row_0/A:col10/1732653663394/Put/seqid=0 2024-11-26T20:41:03,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741969_1145 (size=9857) 2024-11-26T20:41:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-26T20:41:03,985 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=693 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/8f2c3512527046aeb50dec2d49551a77 2024-11-26T20:41:03,994 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/e880c81aaa7a4968bba422542da75437 is 50, key is test_row_0/B:col10/1732653663394/Put/seqid=0 2024-11-26T20:41:04,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741970_1146 (size=9857) 2024-11-26T20:41:04,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-26T20:41:04,402 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=693 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/e880c81aaa7a4968bba422542da75437 2024-11-26T20:41:04,413 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/d6a68e8eaa034e1096f1fdd7e388f0a0 is 50, key is test_row_0/C:col10/1732653663394/Put/seqid=0 2024-11-26T20:41:04,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741971_1147 (size=9857) 2024-11-26T20:41:04,421 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=693 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/d6a68e8eaa034e1096f1fdd7e388f0a0 2024-11-26T20:41:04,427 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/A/8f2c3512527046aeb50dec2d49551a77 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/8f2c3512527046aeb50dec2d49551a77 2024-11-26T20:41:04,432 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/8f2c3512527046aeb50dec2d49551a77, entries=100, sequenceid=693, filesize=9.6 K 2024-11-26T20:41:04,434 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/B/e880c81aaa7a4968bba422542da75437 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/e880c81aaa7a4968bba422542da75437 2024-11-26T20:41:04,438 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/e880c81aaa7a4968bba422542da75437, entries=100, sequenceid=693, filesize=9.6 K 2024-11-26T20:41:04,439 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/.tmp/C/d6a68e8eaa034e1096f1fdd7e388f0a0 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/d6a68e8eaa034e1096f1fdd7e388f0a0 2024-11-26T20:41:04,444 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/d6a68e8eaa034e1096f1fdd7e388f0a0, entries=100, sequenceid=693, filesize=9.6 K 2024-11-26T20:41:04,445 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f47d720597d62d3cb60d84298295c6f3 in 868ms, sequenceid=693, compaction requested=true 2024-11-26T20:41:04,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/4d29a0b523a44837a441ddcf994d60f4, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/9eaab4d6f05a4df49904b3e3a9e53a24, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/4c477350e95f473bb56f7b7a785d0b0d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/4f59f5c9dd8041fbb21d3fda63fd58d6, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/de7ec32aa945426795eb33c4120a66da, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/293567408a58489394fc540beb11df43, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/b4de1e1eec514ba686918ad646144ca9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/e8791a965c764f93956b8636903bc09c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/0578b0f01d6c456d820c536eb5077908, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/e7d83af8a1c649f3a7c32a04372534b7, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/d6436d81097f458cb4477968631a4712, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/3ddeb9baf33446b688bbec2b3faedbd3, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/c4ce64092d964d17867a329d9554ebdf, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/0cfaf324f9fe457fa78113fe5c6132d3, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/7a1a6bfce38a480caddda815a54c72f8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/7dcbb82a933c49ea9cac437c68e8888e, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/bb38e15477e74762ba301d1731a69bb0, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/d2a151f4cf7c4c74a9f13d31bf5a3726, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/9ee0bae0bb3c46fc8481a6bca6a0679e, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f9eb5fa231a444f689b940dd457ec718, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/047c599dfbc849cdabb3fd0444fafab3, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/8e417d22f6c143ff8bf4bb0a4c41638c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/e6bf7b1c84554044b0f4100117974770, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/44ca703211494030955cb3ba88346508, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/30ca14fc312b418cba15e3db57692e52, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/9d448b4705a644b3a642e5c9866775eb, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/2e243992532c47f6af06820c3f2e96dc, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/c940e005e4ae4306a9a474e1f1aba6dd, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/907e89ea5a964a0cb6811fada28733cc, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/a2846680ec194cbfa642b6eae82823a2, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/c7d63a8494d14766b5e4fe5e879974df, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/9f8db60f0565406e8927349f98b24b2b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f76549b5885d47159f6548cdd763048f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f4b6b685298b4af684db7a7e43708b6a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/d7c5051f2c814d52842cc2045e0221ec, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/1c0483ea37cd4c129935c2388af63f87, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/1173ae76ff164d98bebddeb63e613d10, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/50c4dd609c55436f9e79339d6fe90982, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/ba03503bff7a4049a57fcdab923156fe, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/fa34a911f0324fa6965aea96cd1b98d2, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f19eb3372d8348a1a7193f8357171daf, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/6034b85693f9486e92438065a5d9e0b0] to archive 2024-11-26T20:41:04,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T20:41:04,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/4d29a0b523a44837a441ddcf994d60f4 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/4d29a0b523a44837a441ddcf994d60f4 2024-11-26T20:41:04,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/9eaab4d6f05a4df49904b3e3a9e53a24 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/9eaab4d6f05a4df49904b3e3a9e53a24 2024-11-26T20:41:04,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/4c477350e95f473bb56f7b7a785d0b0d to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/4c477350e95f473bb56f7b7a785d0b0d 2024-11-26T20:41:04,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/4f59f5c9dd8041fbb21d3fda63fd58d6 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/4f59f5c9dd8041fbb21d3fda63fd58d6 2024-11-26T20:41:04,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/de7ec32aa945426795eb33c4120a66da to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/de7ec32aa945426795eb33c4120a66da 2024-11-26T20:41:04,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/293567408a58489394fc540beb11df43 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/293567408a58489394fc540beb11df43 2024-11-26T20:41:04,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/b4de1e1eec514ba686918ad646144ca9 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/b4de1e1eec514ba686918ad646144ca9 2024-11-26T20:41:04,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/e8791a965c764f93956b8636903bc09c to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/e8791a965c764f93956b8636903bc09c 2024-11-26T20:41:04,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/0578b0f01d6c456d820c536eb5077908 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/0578b0f01d6c456d820c536eb5077908 2024-11-26T20:41:04,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/e7d83af8a1c649f3a7c32a04372534b7 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/e7d83af8a1c649f3a7c32a04372534b7 2024-11-26T20:41:04,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/d6436d81097f458cb4477968631a4712 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/d6436d81097f458cb4477968631a4712 2024-11-26T20:41:04,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/3ddeb9baf33446b688bbec2b3faedbd3 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/3ddeb9baf33446b688bbec2b3faedbd3 2024-11-26T20:41:04,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/c4ce64092d964d17867a329d9554ebdf to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/c4ce64092d964d17867a329d9554ebdf 2024-11-26T20:41:04,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/0cfaf324f9fe457fa78113fe5c6132d3 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/0cfaf324f9fe457fa78113fe5c6132d3 2024-11-26T20:41:04,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/7a1a6bfce38a480caddda815a54c72f8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/7a1a6bfce38a480caddda815a54c72f8 2024-11-26T20:41:04,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/7dcbb82a933c49ea9cac437c68e8888e to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/7dcbb82a933c49ea9cac437c68e8888e 2024-11-26T20:41:04,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/bb38e15477e74762ba301d1731a69bb0 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/bb38e15477e74762ba301d1731a69bb0 2024-11-26T20:41:04,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/d2a151f4cf7c4c74a9f13d31bf5a3726 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/d2a151f4cf7c4c74a9f13d31bf5a3726 2024-11-26T20:41:04,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/9ee0bae0bb3c46fc8481a6bca6a0679e to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/9ee0bae0bb3c46fc8481a6bca6a0679e 2024-11-26T20:41:04,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f9eb5fa231a444f689b940dd457ec718 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f9eb5fa231a444f689b940dd457ec718 2024-11-26T20:41:04,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/047c599dfbc849cdabb3fd0444fafab3 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/047c599dfbc849cdabb3fd0444fafab3 2024-11-26T20:41:04,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/8e417d22f6c143ff8bf4bb0a4c41638c to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/8e417d22f6c143ff8bf4bb0a4c41638c 2024-11-26T20:41:04,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/e6bf7b1c84554044b0f4100117974770 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/e6bf7b1c84554044b0f4100117974770 2024-11-26T20:41:04,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/44ca703211494030955cb3ba88346508 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/44ca703211494030955cb3ba88346508 2024-11-26T20:41:04,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/30ca14fc312b418cba15e3db57692e52 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/30ca14fc312b418cba15e3db57692e52 2024-11-26T20:41:04,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/9d448b4705a644b3a642e5c9866775eb to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/9d448b4705a644b3a642e5c9866775eb 2024-11-26T20:41:04,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/2e243992532c47f6af06820c3f2e96dc to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/2e243992532c47f6af06820c3f2e96dc 2024-11-26T20:41:04,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/c940e005e4ae4306a9a474e1f1aba6dd to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/c940e005e4ae4306a9a474e1f1aba6dd 2024-11-26T20:41:04,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/907e89ea5a964a0cb6811fada28733cc to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/907e89ea5a964a0cb6811fada28733cc 2024-11-26T20:41:04,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/a2846680ec194cbfa642b6eae82823a2 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/a2846680ec194cbfa642b6eae82823a2 2024-11-26T20:41:04,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/c7d63a8494d14766b5e4fe5e879974df to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/c7d63a8494d14766b5e4fe5e879974df 2024-11-26T20:41:04,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/9f8db60f0565406e8927349f98b24b2b to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/9f8db60f0565406e8927349f98b24b2b 2024-11-26T20:41:04,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f76549b5885d47159f6548cdd763048f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f76549b5885d47159f6548cdd763048f 2024-11-26T20:41:04,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f4b6b685298b4af684db7a7e43708b6a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f4b6b685298b4af684db7a7e43708b6a 2024-11-26T20:41:04,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/d7c5051f2c814d52842cc2045e0221ec to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/d7c5051f2c814d52842cc2045e0221ec 2024-11-26T20:41:04,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/1c0483ea37cd4c129935c2388af63f87 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/1c0483ea37cd4c129935c2388af63f87 2024-11-26T20:41:04,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/1173ae76ff164d98bebddeb63e613d10 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/1173ae76ff164d98bebddeb63e613d10 2024-11-26T20:41:04,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/50c4dd609c55436f9e79339d6fe90982 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/50c4dd609c55436f9e79339d6fe90982 2024-11-26T20:41:04,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/ba03503bff7a4049a57fcdab923156fe to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/ba03503bff7a4049a57fcdab923156fe 2024-11-26T20:41:04,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/fa34a911f0324fa6965aea96cd1b98d2 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/fa34a911f0324fa6965aea96cd1b98d2 2024-11-26T20:41:04,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f19eb3372d8348a1a7193f8357171daf to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/f19eb3372d8348a1a7193f8357171daf 2024-11-26T20:41:04,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/6034b85693f9486e92438065a5d9e0b0 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/6034b85693f9486e92438065a5d9e0b0 2024-11-26T20:41:04,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/7c13c90fa60a476eb6f6a9edeb5f9e67, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/3b8351a31f354ff89d2d604e4ddf3ea8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/5494621c2c0942d1b1931e17e3c00387, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/21b99bbff1e343fc8d53cdd55a99193f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/54c53dd1aa7e40009f5b223b2747db35, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/c3723845162d44a593bce0036d6c4aa9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/f5ae4432a7f84606bf4343409d9ff292, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/be03102a6eb34b7abbaccd33d169bff5, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/24b5cbe4151a4f4fa05ba8089aa9d086, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/c27fb7b6857c45e9b87d3952b89a16f6, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/cce7820acd5a4fb2b208f25079a950b5, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/5e255e7decd841348b83af200f4efc24, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/28132a33635c4a798364c8bc18c4689e, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/da4c876fdd604f2da5e6b57bf5e5e209, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/387567e5227f42e08ebf0d5ee62c2eb8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/d4e96f8395ef450997426008daacce16, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/0c5d227630e0446cb097de369f80375f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/22aabf17fb3d4e8896a2e9f01167dfda, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/c9293b614704450ba84b98eb66eac702, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/b6705fb850934563873dd0284e993b8c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/c20651746214400ba92cb5b9c372ff97, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/6ce15f09a12f408eab5f23e5a949cbdf, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/1f88b068eead4982a559c825ddfc19e5, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/b82bacfaa64a4e0981f8f6de59d6c523, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/8c01b2fe1afe45588cd25c7dd2ffe2f9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/fa7e790088374400b8d0bd0f979d3a3e, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/5679d7756eed40eea04895f3a09c210d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/0486068531424ebe8526b201f7b612a8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/a5bf62ebbf444e53bfba0b78696177a5, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/d49101a8921f4158a69b854e7fd0d1bd, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/17c01b77a0074cd982102292e27a3e1f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/665b4d455b9b4cafb59b92fde314a766, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/172f1a65a5c742d4a94f107b05a1c549, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/bebbbf123ed54ae4be838d4702bef2b1, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/7e37ab7c9ea04e64b4182b2e2f4e680a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/20e1a87b5adf43b38c39a1f49b98914f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/87607341c5ba4ec480aa2120536fc263, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/d526f38ef81d4c0682c7e0867438f601, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/24eb0d05f03f4d18b10314861ca2c45a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/4341217089ed4f598a0fd595fe48ea11, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/4a5fd8674e8b41c1aba642172bb9848f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/5b0f9737e92343ba9be75f61f13d148b] to archive 2024-11-26T20:41:04,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T20:41:04,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/7c13c90fa60a476eb6f6a9edeb5f9e67 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/7c13c90fa60a476eb6f6a9edeb5f9e67 2024-11-26T20:41:04,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/3b8351a31f354ff89d2d604e4ddf3ea8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/3b8351a31f354ff89d2d604e4ddf3ea8 2024-11-26T20:41:04,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/5494621c2c0942d1b1931e17e3c00387 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/5494621c2c0942d1b1931e17e3c00387 2024-11-26T20:41:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-26T20:41:04,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/21b99bbff1e343fc8d53cdd55a99193f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/21b99bbff1e343fc8d53cdd55a99193f 2024-11-26T20:41:04,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/54c53dd1aa7e40009f5b223b2747db35 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/54c53dd1aa7e40009f5b223b2747db35 2024-11-26T20:41:04,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/c3723845162d44a593bce0036d6c4aa9 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/c3723845162d44a593bce0036d6c4aa9 2024-11-26T20:41:04,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/f5ae4432a7f84606bf4343409d9ff292 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/f5ae4432a7f84606bf4343409d9ff292 2024-11-26T20:41:04,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/be03102a6eb34b7abbaccd33d169bff5 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/be03102a6eb34b7abbaccd33d169bff5 2024-11-26T20:41:04,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/24b5cbe4151a4f4fa05ba8089aa9d086 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/24b5cbe4151a4f4fa05ba8089aa9d086 2024-11-26T20:41:04,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/c27fb7b6857c45e9b87d3952b89a16f6 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/c27fb7b6857c45e9b87d3952b89a16f6 2024-11-26T20:41:04,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/cce7820acd5a4fb2b208f25079a950b5 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/cce7820acd5a4fb2b208f25079a950b5 2024-11-26T20:41:04,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/5e255e7decd841348b83af200f4efc24 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/5e255e7decd841348b83af200f4efc24 2024-11-26T20:41:04,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/28132a33635c4a798364c8bc18c4689e to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/28132a33635c4a798364c8bc18c4689e 2024-11-26T20:41:04,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/da4c876fdd604f2da5e6b57bf5e5e209 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/da4c876fdd604f2da5e6b57bf5e5e209 2024-11-26T20:41:04,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/387567e5227f42e08ebf0d5ee62c2eb8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/387567e5227f42e08ebf0d5ee62c2eb8 2024-11-26T20:41:04,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/d4e96f8395ef450997426008daacce16 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/d4e96f8395ef450997426008daacce16 2024-11-26T20:41:04,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/0c5d227630e0446cb097de369f80375f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/0c5d227630e0446cb097de369f80375f 2024-11-26T20:41:04,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/22aabf17fb3d4e8896a2e9f01167dfda to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/22aabf17fb3d4e8896a2e9f01167dfda 2024-11-26T20:41:04,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/c9293b614704450ba84b98eb66eac702 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/c9293b614704450ba84b98eb66eac702 2024-11-26T20:41:04,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/b6705fb850934563873dd0284e993b8c to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/b6705fb850934563873dd0284e993b8c 2024-11-26T20:41:04,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/c20651746214400ba92cb5b9c372ff97 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/c20651746214400ba92cb5b9c372ff97 2024-11-26T20:41:04,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/6ce15f09a12f408eab5f23e5a949cbdf to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/6ce15f09a12f408eab5f23e5a949cbdf 2024-11-26T20:41:04,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/1f88b068eead4982a559c825ddfc19e5 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/1f88b068eead4982a559c825ddfc19e5 2024-11-26T20:41:04,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/b82bacfaa64a4e0981f8f6de59d6c523 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/b82bacfaa64a4e0981f8f6de59d6c523 2024-11-26T20:41:04,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/8c01b2fe1afe45588cd25c7dd2ffe2f9 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/8c01b2fe1afe45588cd25c7dd2ffe2f9 2024-11-26T20:41:04,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/fa7e790088374400b8d0bd0f979d3a3e to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/fa7e790088374400b8d0bd0f979d3a3e 2024-11-26T20:41:04,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/5679d7756eed40eea04895f3a09c210d to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/5679d7756eed40eea04895f3a09c210d 2024-11-26T20:41:04,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/0486068531424ebe8526b201f7b612a8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/0486068531424ebe8526b201f7b612a8 2024-11-26T20:41:04,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/a5bf62ebbf444e53bfba0b78696177a5 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/a5bf62ebbf444e53bfba0b78696177a5 2024-11-26T20:41:04,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/d49101a8921f4158a69b854e7fd0d1bd to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/d49101a8921f4158a69b854e7fd0d1bd 2024-11-26T20:41:04,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/17c01b77a0074cd982102292e27a3e1f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/17c01b77a0074cd982102292e27a3e1f 2024-11-26T20:41:04,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/665b4d455b9b4cafb59b92fde314a766 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/665b4d455b9b4cafb59b92fde314a766 2024-11-26T20:41:04,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/172f1a65a5c742d4a94f107b05a1c549 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/172f1a65a5c742d4a94f107b05a1c549 2024-11-26T20:41:04,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/bebbbf123ed54ae4be838d4702bef2b1 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/bebbbf123ed54ae4be838d4702bef2b1 2024-11-26T20:41:04,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/7e37ab7c9ea04e64b4182b2e2f4e680a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/7e37ab7c9ea04e64b4182b2e2f4e680a 2024-11-26T20:41:04,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/20e1a87b5adf43b38c39a1f49b98914f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/20e1a87b5adf43b38c39a1f49b98914f 2024-11-26T20:41:04,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/87607341c5ba4ec480aa2120536fc263 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/87607341c5ba4ec480aa2120536fc263 2024-11-26T20:41:04,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/d526f38ef81d4c0682c7e0867438f601 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/d526f38ef81d4c0682c7e0867438f601 2024-11-26T20:41:04,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/24eb0d05f03f4d18b10314861ca2c45a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/24eb0d05f03f4d18b10314861ca2c45a 2024-11-26T20:41:04,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/4341217089ed4f598a0fd595fe48ea11 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/4341217089ed4f598a0fd595fe48ea11 2024-11-26T20:41:04,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/4a5fd8674e8b41c1aba642172bb9848f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/4a5fd8674e8b41c1aba642172bb9848f 2024-11-26T20:41:04,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/5b0f9737e92343ba9be75f61f13d148b to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/5b0f9737e92343ba9be75f61f13d148b 2024-11-26T20:41:04,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/22787a0caeb64567980af6e931953160, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/a325a8a70d2b417b9eed2e9a864a2326, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/b8a028dfc12b42ffb7208dd8a2dcd250, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/bca845eaef9f4f5d9c3fe25bce0adf2b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/bcad5ecac0954d25897d7a18d2cea7fb, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/f967319e74fe47ed82e3af9f4f3372c6, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/20a949829ed14d0292e436b308297196, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/1db4b4634cd8406e9236da79767f714d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/9e3dc9f2d29246768d1b02bfdce972fc, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/8aa778824197454ab7210dd6490d514f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/362580f0d5cc4010a70a96e7ca2f8f5e, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/6a933192b3524e7c918642c62dc5a109, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/2f13ac24c6a94bac8a4bf6e42ad0e7da, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/c0b12fab54e84ee3a2f04bc2c3f714d8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/94666524e3ee4f57b349f03fd9684930, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/2cca7068580042d6a65b4f05fa909390, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/e1d97858211445bcad600b52ce1a9ae3, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/d813761b20334e5fb3086c73dec4739a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/6d1b11f293e742299a4ce834efb3c9c0, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/49016f3596df49f5aa53ff60fcebe332, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/11adc336118542dfaee5e50ff42a637c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/559715dbf52b48c3bc04c307f1a95aba, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/d6b38bda5a9b42a4a8a9d397b3a4d66e, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/2f4db612d6564e5092f45e737ebdc7fc, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/6a768fb48a48473db1cab0c7a881c92a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/919dce535d494e15a8b7b228ebf8b2b2, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/be958d88955a4164b98db8cb5a8fddee, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/ebd45015343a4dd7847fb03f78208e69, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/0af7a0e1c99e412ca3d6a435ebff937b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/f1e3399c4bd44247ae636204d3cab73b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/ca5aaa5d33244f10a6d2e3bf63a74298, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/dd1ed13d87ee464d85c662227e6018eb, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/fd7bd01486fb415cb555bbe2426bd503, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/f22ee3abc9de46c5a641f830c4bef440, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/37d08d0354c14b0b84396384a38cb1f5, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/5d1e0bffd7de4529ad9ad40f87f0748f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/85e7158991674c14a3cd98a7556c7170, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/c0f33cbdf5784fa5808ba6d430fbd1b3, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/9525f41f8bb749b7aa8aecabf7694ebd, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/d3cb8fd2cf04459ab967c601f28cc368, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/de707e004bb5441f83dd0bfae878e93a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/bf2f267646254d0da31c56f1d0477763] to archive 2024-11-26T20:41:04,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T20:41:04,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/22787a0caeb64567980af6e931953160 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/22787a0caeb64567980af6e931953160 2024-11-26T20:41:04,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/a325a8a70d2b417b9eed2e9a864a2326 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/a325a8a70d2b417b9eed2e9a864a2326 2024-11-26T20:41:04,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/b8a028dfc12b42ffb7208dd8a2dcd250 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/b8a028dfc12b42ffb7208dd8a2dcd250 2024-11-26T20:41:04,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/bca845eaef9f4f5d9c3fe25bce0adf2b to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/bca845eaef9f4f5d9c3fe25bce0adf2b 2024-11-26T20:41:04,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/bcad5ecac0954d25897d7a18d2cea7fb to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/bcad5ecac0954d25897d7a18d2cea7fb 2024-11-26T20:41:04,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/f967319e74fe47ed82e3af9f4f3372c6 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/f967319e74fe47ed82e3af9f4f3372c6 2024-11-26T20:41:04,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/20a949829ed14d0292e436b308297196 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/20a949829ed14d0292e436b308297196 2024-11-26T20:41:04,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/1db4b4634cd8406e9236da79767f714d to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/1db4b4634cd8406e9236da79767f714d 2024-11-26T20:41:04,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/9e3dc9f2d29246768d1b02bfdce972fc to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/9e3dc9f2d29246768d1b02bfdce972fc 2024-11-26T20:41:04,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/8aa778824197454ab7210dd6490d514f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/8aa778824197454ab7210dd6490d514f 2024-11-26T20:41:04,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/362580f0d5cc4010a70a96e7ca2f8f5e to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/362580f0d5cc4010a70a96e7ca2f8f5e 2024-11-26T20:41:04,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/6a933192b3524e7c918642c62dc5a109 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/6a933192b3524e7c918642c62dc5a109 2024-11-26T20:41:04,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/2f13ac24c6a94bac8a4bf6e42ad0e7da to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/2f13ac24c6a94bac8a4bf6e42ad0e7da 2024-11-26T20:41:04,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/c0b12fab54e84ee3a2f04bc2c3f714d8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/c0b12fab54e84ee3a2f04bc2c3f714d8 2024-11-26T20:41:04,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/94666524e3ee4f57b349f03fd9684930 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/94666524e3ee4f57b349f03fd9684930 2024-11-26T20:41:04,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/2cca7068580042d6a65b4f05fa909390 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/2cca7068580042d6a65b4f05fa909390 2024-11-26T20:41:04,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/e1d97858211445bcad600b52ce1a9ae3 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/e1d97858211445bcad600b52ce1a9ae3 2024-11-26T20:41:04,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/d813761b20334e5fb3086c73dec4739a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/d813761b20334e5fb3086c73dec4739a 2024-11-26T20:41:04,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/6d1b11f293e742299a4ce834efb3c9c0 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/6d1b11f293e742299a4ce834efb3c9c0 2024-11-26T20:41:04,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/49016f3596df49f5aa53ff60fcebe332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/49016f3596df49f5aa53ff60fcebe332 2024-11-26T20:41:04,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/11adc336118542dfaee5e50ff42a637c to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/11adc336118542dfaee5e50ff42a637c 2024-11-26T20:41:04,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/559715dbf52b48c3bc04c307f1a95aba to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/559715dbf52b48c3bc04c307f1a95aba 2024-11-26T20:41:04,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/d6b38bda5a9b42a4a8a9d397b3a4d66e to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/d6b38bda5a9b42a4a8a9d397b3a4d66e 2024-11-26T20:41:04,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/2f4db612d6564e5092f45e737ebdc7fc to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/2f4db612d6564e5092f45e737ebdc7fc 2024-11-26T20:41:04,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/6a768fb48a48473db1cab0c7a881c92a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/6a768fb48a48473db1cab0c7a881c92a 2024-11-26T20:41:04,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/919dce535d494e15a8b7b228ebf8b2b2 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/919dce535d494e15a8b7b228ebf8b2b2 2024-11-26T20:41:04,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/be958d88955a4164b98db8cb5a8fddee to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/be958d88955a4164b98db8cb5a8fddee 2024-11-26T20:41:04,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/ebd45015343a4dd7847fb03f78208e69 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/ebd45015343a4dd7847fb03f78208e69 2024-11-26T20:41:04,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/0af7a0e1c99e412ca3d6a435ebff937b to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/0af7a0e1c99e412ca3d6a435ebff937b 2024-11-26T20:41:04,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/f1e3399c4bd44247ae636204d3cab73b to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/f1e3399c4bd44247ae636204d3cab73b 2024-11-26T20:41:04,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/ca5aaa5d33244f10a6d2e3bf63a74298 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/ca5aaa5d33244f10a6d2e3bf63a74298 2024-11-26T20:41:04,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/dd1ed13d87ee464d85c662227e6018eb to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/dd1ed13d87ee464d85c662227e6018eb 2024-11-26T20:41:04,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/fd7bd01486fb415cb555bbe2426bd503 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/fd7bd01486fb415cb555bbe2426bd503 2024-11-26T20:41:04,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/f22ee3abc9de46c5a641f830c4bef440 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/f22ee3abc9de46c5a641f830c4bef440 2024-11-26T20:41:04,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/37d08d0354c14b0b84396384a38cb1f5 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/37d08d0354c14b0b84396384a38cb1f5 2024-11-26T20:41:04,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/5d1e0bffd7de4529ad9ad40f87f0748f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/5d1e0bffd7de4529ad9ad40f87f0748f 2024-11-26T20:41:04,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/85e7158991674c14a3cd98a7556c7170 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/85e7158991674c14a3cd98a7556c7170 2024-11-26T20:41:04,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/c0f33cbdf5784fa5808ba6d430fbd1b3 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/c0f33cbdf5784fa5808ba6d430fbd1b3 2024-11-26T20:41:04,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/9525f41f8bb749b7aa8aecabf7694ebd to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/9525f41f8bb749b7aa8aecabf7694ebd 2024-11-26T20:41:04,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/d3cb8fd2cf04459ab967c601f28cc368 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/d3cb8fd2cf04459ab967c601f28cc368 2024-11-26T20:41:04,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/de707e004bb5441f83dd0bfae878e93a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/de707e004bb5441f83dd0bfae878e93a 2024-11-26T20:41:04,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/bf2f267646254d0da31c56f1d0477763 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/bf2f267646254d0da31c56f1d0477763 2024-11-26T20:41:04,690 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/recovered.edits/696.seqid, newMaxSeqId=696, maxSeqId=1 2024-11-26T20:41:04,691 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. 2024-11-26T20:41:04,691 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for f47d720597d62d3cb60d84298295c6f3: Waiting for close lock at 1732653663576Running coprocessor pre-close hooks at 1732653663576Disabling compacts and flushes for region at 1732653663576Disabling writes for close at 1732653663577 (+1 ms)Obtaining lock to block concurrent updates at 1732653663577Preparing flush snapshotting stores in f47d720597d62d3cb60d84298295c6f3 at 1732653663577Finished memstore snapshotting TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732653663577Flushing stores of TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3. at 1732653663578 (+1 ms)Flushing f47d720597d62d3cb60d84298295c6f3/A: creating writer at 1732653663578Flushing f47d720597d62d3cb60d84298295c6f3/A: appending metadata at 1732653663581 (+3 ms)Flushing f47d720597d62d3cb60d84298295c6f3/A: closing flushed file at 1732653663581Flushing f47d720597d62d3cb60d84298295c6f3/B: creating writer at 1732653663990 (+409 ms)Flushing f47d720597d62d3cb60d84298295c6f3/B: appending metadata at 1732653663993 (+3 ms)Flushing f47d720597d62d3cb60d84298295c6f3/B: closing flushed file at 1732653663993Flushing f47d720597d62d3cb60d84298295c6f3/C: creating writer at 1732653664408 (+415 ms)Flushing f47d720597d62d3cb60d84298295c6f3/C: appending metadata at 1732653664412 (+4 ms)Flushing f47d720597d62d3cb60d84298295c6f3/C: closing flushed file at 1732653664412Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54505ef8: reopening flushed file at 1732653664426 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fc470c7: reopening flushed file at 1732653664433 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@81399c: reopening flushed file at 1732653664439 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f47d720597d62d3cb60d84298295c6f3 in 868ms, sequenceid=693, compaction requested=true at 1732653664445 (+6 ms)Writing region close event to WAL at 1732653664686 (+241 ms)Running coprocessor post-close hooks at 1732653664690 (+4 ms)Closed at 1732653664690 2024-11-26T20:41:04,693 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:41:04,694 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=f47d720597d62d3cb60d84298295c6f3, regionState=CLOSED 2024-11-26T20:41:04,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 because future has completed 2024-11-26T20:41:04,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-26T20:41:04,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure f47d720597d62d3cb60d84298295c6f3, server=cd6be161543e,40291,1732653631795 in 1.2780 sec 2024-11-26T20:41:04,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-26T20:41:04,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f47d720597d62d3cb60d84298295c6f3, UNASSIGN in 1.2840 sec 2024-11-26T20:41:04,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-26T20:41:04,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.2900 sec 2024-11-26T20:41:04,711 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653664711"}]},"ts":"1732653664711"} 2024-11-26T20:41:04,713 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T20:41:04,713 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T20:41:04,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3120 sec 2024-11-26T20:41:05,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-26T20:41:05,555 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:05,559 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T20:41:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:41:05,567 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=37, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:41:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-26T20:41:05,569 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=37, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:41:05,580 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:41:05,588 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A, FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B, FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C, FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/recovered.edits] 2024-11-26T20:41:05,592 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/806989e743ef424dae12e7934589d0e7 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/806989e743ef424dae12e7934589d0e7 2024-11-26T20:41:05,596 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/8f2c3512527046aeb50dec2d49551a77 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/8f2c3512527046aeb50dec2d49551a77 2024-11-26T20:41:05,597 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/bdb8ea1d7835416f99e0e520ee6b882b to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/A/bdb8ea1d7835416f99e0e520ee6b882b 2024-11-26T20:41:05,601 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/16dd20098bc940a48ad7afecda1bc558 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/16dd20098bc940a48ad7afecda1bc558 2024-11-26T20:41:05,603 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/6fff4ae6ba2e4e0c8ef59377618d498e to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/6fff4ae6ba2e4e0c8ef59377618d498e 2024-11-26T20:41:05,605 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/e880c81aaa7a4968bba422542da75437 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/B/e880c81aaa7a4968bba422542da75437 2024-11-26T20:41:05,615 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/8c0bb1edd15746318fac86d3fcaeafc9 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/8c0bb1edd15746318fac86d3fcaeafc9 2024-11-26T20:41:05,616 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/d6a68e8eaa034e1096f1fdd7e388f0a0 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/d6a68e8eaa034e1096f1fdd7e388f0a0 2024-11-26T20:41:05,619 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/dc97731b37bc47f59461109352b8f0ba to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/C/dc97731b37bc47f59461109352b8f0ba 2024-11-26T20:41:05,623 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/recovered.edits/696.seqid to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3/recovered.edits/696.seqid 2024-11-26T20:41:05,623 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/f47d720597d62d3cb60d84298295c6f3 2024-11-26T20:41:05,624 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T20:41:05,631 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=37, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:41:05,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-26T20:41:05,644 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T20:41:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-26T20:41:05,679 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T20:41:05,680 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=37, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:41:05,680 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T20:41:05,681 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732653665681"}]},"ts":"9223372036854775807"} 2024-11-26T20:41:05,685 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T20:41:05,685 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f47d720597d62d3cb60d84298295c6f3, NAME => 'TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T20:41:05,685 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T20:41:05,685 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732653665685"}]},"ts":"9223372036854775807"} 2024-11-26T20:41:05,688 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T20:41:05,689 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=37, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:41:05,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 129 msec 2024-11-26T20:41:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-26T20:41:05,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T20:41:05,885 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:05,895 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/cd6be161543e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;cd6be161543e:40291-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1445524182_22 at /127.0.0.1:52304 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=439 (was 343) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5848 (was 6533) 2024-11-26T20:41:05,904 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=439, ProcessCount=11, AvailableMemoryMB=5848 2024-11-26T20:41:05,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T20:41:05,906 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T20:41:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T20:41:05,908 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T20:41:05,908 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:05,908 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 38 2024-11-26T20:41:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T20:41:05,909 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T20:41:05,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741972_1148 (size=1061) 2024-11-26T20:41:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T20:41:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T20:41:06,318 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794 2024-11-26T20:41:06,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741973_1149 (size=53) 2024-11-26T20:41:06,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:41:06,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 00a77bb01b2de4ac52368f2c3e602b3a, disabling compactions & flushes 2024-11-26T20:41:06,333 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:06,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:06,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. after waiting 0 ms 2024-11-26T20:41:06,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:06,333 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:06,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 00a77bb01b2de4ac52368f2c3e602b3a: Waiting for close lock at 1732653666333Disabling compacts and flushes for region at 1732653666333Disabling writes for close at 1732653666333Writing region close event to WAL at 1732653666333Closed at 1732653666333 2024-11-26T20:41:06,335 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T20:41:06,335 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732653666335"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732653666335"}]},"ts":"1732653666335"} 2024-11-26T20:41:06,338 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T20:41:06,339 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T20:41:06,340 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653666339"}]},"ts":"1732653666339"} 2024-11-26T20:41:06,342 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T20:41:06,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a77bb01b2de4ac52368f2c3e602b3a, ASSIGN}] 2024-11-26T20:41:06,343 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a77bb01b2de4ac52368f2c3e602b3a, ASSIGN 2024-11-26T20:41:06,344 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a77bb01b2de4ac52368f2c3e602b3a, ASSIGN; state=OFFLINE, location=cd6be161543e,40291,1732653631795; forceNewPlan=false, retain=false 2024-11-26T20:41:06,495 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=00a77bb01b2de4ac52368f2c3e602b3a, regionState=OPENING, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:41:06,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a77bb01b2de4ac52368f2c3e602b3a, ASSIGN because future has completed 2024-11-26T20:41:06,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795}] 2024-11-26T20:41:06,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T20:41:06,653 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:06,653 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:41:06,654 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:06,654 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:41:06,654 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:06,654 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:06,655 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:06,656 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:41:06,657 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00a77bb01b2de4ac52368f2c3e602b3a columnFamilyName A 2024-11-26T20:41:06,657 DEBUG [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:06,658 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] regionserver.HStore(327): Store=00a77bb01b2de4ac52368f2c3e602b3a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:41:06,658 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:06,659 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:41:06,659 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00a77bb01b2de4ac52368f2c3e602b3a columnFamilyName B 2024-11-26T20:41:06,659 DEBUG [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:06,660 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] regionserver.HStore(327): Store=00a77bb01b2de4ac52368f2c3e602b3a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:41:06,660 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:06,661 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:41:06,661 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00a77bb01b2de4ac52368f2c3e602b3a columnFamilyName C 2024-11-26T20:41:06,661 DEBUG [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:06,661 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] regionserver.HStore(327): Store=00a77bb01b2de4ac52368f2c3e602b3a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:41:06,661 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:06,661 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:06,662 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:06,662 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:06,663 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:06,663 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:06,664 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T20:41:06,665 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:06,667 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:41:06,667 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 00a77bb01b2de4ac52368f2c3e602b3a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74041694, jitterRate=0.10330721735954285}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T20:41:06,668 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:06,669 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 00a77bb01b2de4ac52368f2c3e602b3a: Running coprocessor pre-open hook at 1732653666654Writing region info on filesystem at 1732653666654Initializing all the Stores at 1732653666655 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653666655Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653666655Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653666655Cleaning up temporary data from old regions at 1732653666663 (+8 ms)Running coprocessor post-open hooks at 1732653666668 (+5 ms)Region opened successfully at 1732653666669 (+1 ms) 2024-11-26T20:41:06,669 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., pid=40, masterSystemTime=1732653666650 2024-11-26T20:41:06,671 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:06,671 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:06,672 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=00a77bb01b2de4ac52368f2c3e602b3a, regionState=OPEN, openSeqNum=2, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:41:06,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 because future has completed 2024-11-26T20:41:06,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-26T20:41:06,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 in 175 msec 2024-11-26T20:41:06,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-26T20:41:06,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a77bb01b2de4ac52368f2c3e602b3a, ASSIGN in 334 msec 2024-11-26T20:41:06,678 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T20:41:06,678 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653666678"}]},"ts":"1732653666678"} 2024-11-26T20:41:06,679 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T20:41:06,680 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T20:41:06,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 774 msec 2024-11-26T20:41:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T20:41:07,044 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:07,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49a00013, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,045 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:41:07,045 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:41:07,047 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:41:07,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:41:07,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:41:07,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f15291f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:41:07,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:41:07,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:07,049 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:41:07,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e548935, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,050 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:41:07,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:41:07,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:41:07,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:41:07,054 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T20:41:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is cd6be161543e,42359,1732653630959 2024-11-26T20:41:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@67a4aeb8 2024-11-26T20:41:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T20:41:07,055 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T20:41:07,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T20:41:07,060 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T20:41:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-26T20:41:07,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741974_1150 (size=1097) 2024-11-26T20:41:07,078 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-26T20:41:07,079 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-26T20:41:07,082 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T20:41:07,088 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a77bb01b2de4ac52368f2c3e602b3a, REOPEN/MOVE}] 2024-11-26T20:41:07,089 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a77bb01b2de4ac52368f2c3e602b3a, REOPEN/MOVE 2024-11-26T20:41:07,089 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=43 updating hbase:meta row=00a77bb01b2de4ac52368f2c3e602b3a, regionState=CLOSING, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:41:07,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a77bb01b2de4ac52368f2c3e602b3a, REOPEN/MOVE because future has completed 2024-11-26T20:41:07,092 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T20:41:07,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795}] 2024-11-26T20:41:07,245 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] handler.UnassignRegionHandler(122): Close 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:07,245 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T20:41:07,245 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1722): Closing 00a77bb01b2de4ac52368f2c3e602b3a, disabling compactions & flushes 2024-11-26T20:41:07,245 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:07,245 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:07,245 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. after waiting 0 ms 2024-11-26T20:41:07,245 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:07,249 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:41:07,250 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:07,250 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1676): Region close journal for 00a77bb01b2de4ac52368f2c3e602b3a: Waiting for close lock at 1732653667245Running coprocessor pre-close hooks at 1732653667245Disabling compacts and flushes for region at 1732653667245Disabling writes for close at 1732653667245Writing region close event to WAL at 1732653667246 (+1 ms)Running coprocessor post-close hooks at 1732653667250 (+4 ms)Closed at 1732653667250 2024-11-26T20:41:07,250 WARN [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegionServer(3299): Not adding moved region record: 00a77bb01b2de4ac52368f2c3e602b3a to self. 2024-11-26T20:41:07,253 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] handler.UnassignRegionHandler(157): Closed 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:07,254 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=43 updating hbase:meta row=00a77bb01b2de4ac52368f2c3e602b3a, regionState=CLOSED 2024-11-26T20:41:07,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 because future has completed 2024-11-26T20:41:07,267 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-11-26T20:41:07,267 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; CloseRegionProcedure 00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 in 172 msec 2024-11-26T20:41:07,268 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a77bb01b2de4ac52368f2c3e602b3a, REOPEN/MOVE; state=CLOSED, location=cd6be161543e,40291,1732653631795; forceNewPlan=false, retain=true 2024-11-26T20:41:07,419 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=43 updating hbase:meta row=00a77bb01b2de4ac52368f2c3e602b3a, regionState=OPENING, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:41:07,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a77bb01b2de4ac52368f2c3e602b3a, REOPEN/MOVE because future has completed 2024-11-26T20:41:07,422 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=43, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795}] 2024-11-26T20:41:07,576 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:07,577 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:41:07,577 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:07,577 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:41:07,577 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:07,577 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:07,580 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:07,580 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:41:07,588 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00a77bb01b2de4ac52368f2c3e602b3a columnFamilyName A 2024-11-26T20:41:07,591 DEBUG [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:07,593 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] regionserver.HStore(327): Store=00a77bb01b2de4ac52368f2c3e602b3a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:41:07,593 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:07,594 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:41:07,594 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00a77bb01b2de4ac52368f2c3e602b3a columnFamilyName B 2024-11-26T20:41:07,594 DEBUG [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:07,595 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] regionserver.HStore(327): Store=00a77bb01b2de4ac52368f2c3e602b3a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:41:07,595 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:07,596 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:41:07,596 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00a77bb01b2de4ac52368f2c3e602b3a columnFamilyName C 2024-11-26T20:41:07,596 DEBUG [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:07,597 INFO [StoreOpener-00a77bb01b2de4ac52368f2c3e602b3a-1 {}] regionserver.HStore(327): Store=00a77bb01b2de4ac52368f2c3e602b3a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:41:07,597 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:07,597 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:07,598 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:07,599 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:07,601 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:07,601 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:07,601 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T20:41:07,603 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:07,604 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 00a77bb01b2de4ac52368f2c3e602b3a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69697524, jitterRate=0.038574039936065674}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T20:41:07,604 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:07,606 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 00a77bb01b2de4ac52368f2c3e602b3a: Running coprocessor pre-open hook at 1732653667578Writing region info on filesystem at 1732653667578Initializing all the Stores at 1732653667578Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653667578Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653667579 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653667579Cleaning up temporary data from old regions at 1732653667601 (+22 ms)Running coprocessor post-open hooks at 1732653667604 (+3 ms)Region opened successfully at 1732653667605 (+1 ms) 2024-11-26T20:41:07,606 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., pid=45, masterSystemTime=1732653667573 2024-11-26T20:41:07,609 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:07,609 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:07,609 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=43 updating hbase:meta row=00a77bb01b2de4ac52368f2c3e602b3a, regionState=OPEN, openSeqNum=5, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:41:07,612 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=43, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 because future has completed 2024-11-26T20:41:07,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=43 2024-11-26T20:41:07,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=43, state=SUCCESS, hasLock=false; OpenRegionProcedure 00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 in 191 msec 2024-11-26T20:41:07,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-26T20:41:07,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a77bb01b2de4ac52368f2c3e602b3a, REOPEN/MOVE in 529 msec 2024-11-26T20:41:07,622 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=41 2024-11-26T20:41:07,622 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=41, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 538 msec 2024-11-26T20:41:07,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 561 msec 2024-11-26T20:41:07,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=41 2024-11-26T20:41:07,628 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:07,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad8b864, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,629 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:41:07,629 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:41:07,630 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:41:07,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:41:07,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:41:07,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55cd97ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:41:07,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:41:07,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:07,633 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:41:07,634 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d9b2e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:41:07,635 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:41:07,635 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:41:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:41:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:41:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12988e5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:41:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:41:07,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:07,637 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:41:07,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c9a4373, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,640 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:41:07,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:41:07,641 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:41:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:41:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:41:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@694b8223, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:41:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:41:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:07,644 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:41:07,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@226edb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,649 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:41:07,649 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:41:07,649 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:41:07,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:41:07,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:41:07,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b23fa1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:41:07,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:41:07,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:07,651 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:41:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f6cabea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,653 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:41:07,653 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:41:07,654 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:41:07,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:41:07,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:41:07,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f00fba7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:41:07,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:41:07,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:07,656 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:41:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c2741a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ffb7a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,657 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:41:07,658 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:41:07,659 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:41:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:41:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:41:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71b9a76f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:41:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:41:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:07,660 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:41:07,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2736c793, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,664 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:41:07,664 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:41:07,665 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:41:07,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:41:07,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:41:07,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699d3459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:41:07,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:41:07,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:07,667 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:41:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@448192b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,668 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:41:07,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:41:07,669 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:41:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:41:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:41:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@237643a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:41:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:41:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:07,676 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:41:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4940a7d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,680 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:41:07,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:41:07,681 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:41:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:41:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:41:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@428569ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:41:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:41:07,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:07,684 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:41:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3649d1c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:07,689 DEBUG [Thread-738 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:41:07,690 DEBUG [Thread-736 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:41:07,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:41:07,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:41:07,693 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:41:07,696 DEBUG [Thread-730 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:41:07,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:41:07,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:41:07,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:41:07,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:41:07,699 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:41:07,699 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:41:07,700 DEBUG [Thread-741 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:41:07,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:41:07,701 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:41:07,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:41:07,703 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:41:07,704 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:41:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:41:07,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:41:07,706 DEBUG [Thread-745 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:41:07,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-26T20:41:07,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:41:07,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:41:07,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:41:07,709 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:41:07,709 DEBUG [Thread-743 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:41:07,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T20:41:07,710 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:41:07,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:41:07,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:41:07,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:41:07,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:41:07,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:41:07,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:41:07,716 DEBUG [Thread-732 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:41:07,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:41:07,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:07,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00a77bb01b2de4ac52368f2c3e602b3a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T20:41:07,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=A 2024-11-26T20:41:07,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:07,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=B 2024-11-26T20:41:07,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:07,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=C 2024-11-26T20:41:07,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:07,722 DEBUG [Thread-732 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:41:07,723 DEBUG [Thread-732 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:41:07,723 DEBUG [Thread-734 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:41:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:41:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:41:07,726 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:41:07,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:41:07,737 DEBUG [Thread-747 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:41:07,738 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:41:07,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:41:07,738 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:41:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:41:07,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:41:07,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:41:07,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126abd5536b141547219a33fdc201b35d26_00a77bb01b2de4ac52368f2c3e602b3a is 50, key is test_row_0/A:col10/1732653667712/Put/seqid=0 2024-11-26T20:41:07,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653667795, max timestamp=1732653667801Num uniques 859; 2024-11-26T20:41:07,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653667795, max timestamp=1732653667801Num uniques 859; 2024-11-26T20:41:07,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:41:07,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:41:07,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653667795, max timestamp=1732653667801Num uniques 859; 2024-11-26T20:41:07,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T20:41:07,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:41:07,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741975_1151 (size=14594) 2024-11-26T20:41:07,828 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:07,833 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126abd5536b141547219a33fdc201b35d26_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126abd5536b141547219a33fdc201b35d26_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:07,834 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/3fd06b044e0d447a8f5b9ad7ebf667cc, store: [table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:07,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/3fd06b044e0d447a8f5b9ad7ebf667cc is 175, key is test_row_0/A:col10/1732653667712/Put/seqid=0 2024-11-26T20:41:07,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T20:41:07,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:07,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:07,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:07,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:07,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:07,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741976_1152 (size=39549) 2024-11-26T20:41:07,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:07,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:07,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55242 deadline: 1732653727852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:07,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:07,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:07,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:07,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:07,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653727854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:07,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:07,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:07,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:07,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55170 deadline: 1732653727866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:07,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:07,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:07,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:07,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:07,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653727870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:07,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:07,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653727872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:08,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:08,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55242 deadline: 1732653727984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:08,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55170 deadline: 1732653727984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:08,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:08,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653727984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:08,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:08,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:08,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:08,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653727994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:08,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:08,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653727996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:08,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T20:41:08,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:08,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:08,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:08,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:08,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:08,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:08,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T20:41:08,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T20:41:08,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:08,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:08,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:08,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:08,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:08,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:08,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:08,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55242 deadline: 1732653728214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:08,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55170 deadline: 1732653728214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:08,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653728215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:08,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653728215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:08,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653728215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:08,271 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/3fd06b044e0d447a8f5b9ad7ebf667cc 2024-11-26T20:41:08,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/9f835546920f41cbb30df4a8d2781032 is 50, key is test_row_0/B:col10/1732653667712/Put/seqid=0 2024-11-26T20:41:08,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T20:41:08,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:08,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:08,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:08,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:08,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:08,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T20:41:08,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741977_1153 (size=12001) 2024-11-26T20:41:08,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/9f835546920f41cbb30df4a8d2781032 2024-11-26T20:41:08,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/29448f06910c4989961d0edca282cb79 is 50, key is test_row_0/C:col10/1732653667712/Put/seqid=0 2024-11-26T20:41:08,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741978_1154 (size=12001) 2024-11-26T20:41:08,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/29448f06910c4989961d0edca282cb79 2024-11-26T20:41:08,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/3fd06b044e0d447a8f5b9ad7ebf667cc as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3fd06b044e0d447a8f5b9ad7ebf667cc 2024-11-26T20:41:08,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3fd06b044e0d447a8f5b9ad7ebf667cc, entries=200, sequenceid=14, filesize=38.6 K 2024-11-26T20:41:08,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/9f835546920f41cbb30df4a8d2781032 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/9f835546920f41cbb30df4a8d2781032 2024-11-26T20:41:08,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/9f835546920f41cbb30df4a8d2781032, entries=150, sequenceid=14, filesize=11.7 K 2024-11-26T20:41:08,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/29448f06910c4989961d0edca282cb79 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/29448f06910c4989961d0edca282cb79 2024-11-26T20:41:08,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/29448f06910c4989961d0edca282cb79, entries=150, sequenceid=14, filesize=11.7 K 2024-11-26T20:41:08,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 00a77bb01b2de4ac52368f2c3e602b3a in 758ms, sequenceid=14, compaction requested=false 2024-11-26T20:41:08,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:08,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T20:41:08,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:08,485 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 00a77bb01b2de4ac52368f2c3e602b3a 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-26T20:41:08,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=A 2024-11-26T20:41:08,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:08,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=B 2024-11-26T20:41:08,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:08,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=C 2024-11-26T20:41:08,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:08,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126844a2ca7fd0d412092b591c64d2976d7_00a77bb01b2de4ac52368f2c3e602b3a is 50, key is test_row_0/A:col10/1732653667866/Put/seqid=0 2024-11-26T20:41:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:08,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:08,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741979_1155 (size=12154) 2024-11-26T20:41:08,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653728535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:08,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:08,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653728535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:08,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:08,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:08,552 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126844a2ca7fd0d412092b591c64d2976d7_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126844a2ca7fd0d412092b591c64d2976d7_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:08,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:08,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55170 deadline: 1732653728536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:08,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:08,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/a97b1c10e3ca4b5dbea59a05a71732a8, store: [table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:08,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/a97b1c10e3ca4b5dbea59a05a71732a8 is 175, key is test_row_0/A:col10/1732653667866/Put/seqid=0 2024-11-26T20:41:08,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:08,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55242 deadline: 1732653728543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:08,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:08,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653728544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:08,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:08,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:08,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741980_1156 (size=30955) 2024-11-26T20:41:08,596 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/a97b1c10e3ca4b5dbea59a05a71732a8 2024-11-26T20:41:08,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/3d25af26302e4bc2a0a9848a21d72c91 is 50, key is test_row_0/B:col10/1732653667866/Put/seqid=0 2024-11-26T20:41:08,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741981_1157 (size=12001) 2024-11-26T20:41:08,646 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/3d25af26302e4bc2a0a9848a21d72c91 2024-11-26T20:41:08,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653728654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653728654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:08,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:08,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55170 deadline: 1732653728664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:08,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:08,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55242 deadline: 1732653728664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:08,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653728664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:08,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:08,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:08,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/af7b01b219d24494a5d16dbb6c4d8a34 is 50, key is test_row_0/C:col10/1732653667866/Put/seqid=0 2024-11-26T20:41:08,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741982_1158 (size=12001) 2024-11-26T20:41:08,680 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/af7b01b219d24494a5d16dbb6c4d8a34 2024-11-26T20:41:08,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/a97b1c10e3ca4b5dbea59a05a71732a8 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/a97b1c10e3ca4b5dbea59a05a71732a8 2024-11-26T20:41:08,693 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/a97b1c10e3ca4b5dbea59a05a71732a8, entries=150, sequenceid=50, filesize=30.2 K 2024-11-26T20:41:08,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/3d25af26302e4bc2a0a9848a21d72c91 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/3d25af26302e4bc2a0a9848a21d72c91 2024-11-26T20:41:08,706 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/3d25af26302e4bc2a0a9848a21d72c91, entries=150, sequenceid=50, filesize=11.7 K 2024-11-26T20:41:08,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/af7b01b219d24494a5d16dbb6c4d8a34 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/af7b01b219d24494a5d16dbb6c4d8a34 2024-11-26T20:41:08,715 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/af7b01b219d24494a5d16dbb6c4d8a34, entries=150, sequenceid=50, filesize=11.7 K 2024-11-26T20:41:08,718 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 00a77bb01b2de4ac52368f2c3e602b3a in 232ms, sequenceid=50, compaction requested=false 2024-11-26T20:41:08,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:08,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:08,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-26T20:41:08,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-26T20:41:08,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-26T20:41:08,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0130 sec 2024-11-26T20:41:08,729 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.0230 sec 2024-11-26T20:41:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T20:41:08,844 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:08,845 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:41:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-26T20:41:08,848 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:41:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T20:41:08,849 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:41:08,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:41:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:08,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00a77bb01b2de4ac52368f2c3e602b3a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T20:41:08,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=A 2024-11-26T20:41:08,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:08,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=B 2024-11-26T20:41:08,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:08,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=C 2024-11-26T20:41:08,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:08,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f78cf6959eb74d079ac287f2465bf8fe_00a77bb01b2de4ac52368f2c3e602b3a is 50, key is test_row_0/A:col10/1732653668542/Put/seqid=0 2024-11-26T20:41:08,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741983_1159 (size=14594) 2024-11-26T20:41:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T20:41:08,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732653668966, max timestamp=1732653668977Num uniques 859; 2024-11-26T20:41:08,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:41:08,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653668966, max timestamp=1732653668977Num uniques 859; 2024-11-26T20:41:08,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:41:08,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653668966, max timestamp=1732653668977Num uniques 859; 2024-11-26T20:41:08,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:41:09,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T20:41:09,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:09,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:09,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:09,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:09,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:09,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653729019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:09,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653729021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:09,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:09,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653729028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:09,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:09,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:09,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55170 deadline: 1732653729029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:09,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55242 deadline: 1732653729030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:09,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:09,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:09,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653729135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:09,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653729135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:09,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:09,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653729135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:09,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:09,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:09,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T20:41:09,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:09,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:09,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:09,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:09,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:09,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55170 deadline: 1732653729154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:09,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:09,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55242 deadline: 1732653729154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T20:41:09,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:09,310 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:09,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T20:41:09,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:09,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:09,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:09,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:09,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:09,316 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f78cf6959eb74d079ac287f2465bf8fe_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f78cf6959eb74d079ac287f2465bf8fe_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:09,317 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/3591f216b9144464bdd9d738f3efa41f, store: [table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:09,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/3591f216b9144464bdd9d738f3efa41f is 175, key is test_row_0/A:col10/1732653668542/Put/seqid=0 2024-11-26T20:41:09,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741984_1160 (size=39549) 2024-11-26T20:41:09,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653729345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:09,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653729345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:09,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:09,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:09,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653729345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:09,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:09,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55170 deadline: 1732653729374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:09,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:09,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55242 deadline: 1732653729374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:09,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:09,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:09,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T20:41:09,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:09,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:09,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:09,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:09,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T20:41:09,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T20:41:09,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:09,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:09,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:09,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:09,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:09,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653729654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:09,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:09,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653729654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:09,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653729654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:09,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55170 deadline: 1732653729684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:09,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55242 deadline: 1732653729684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:09,724 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/3591f216b9144464bdd9d738f3efa41f 2024-11-26T20:41:09,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/b62a41f4b859410f9e00c4425c735415 is 50, key is test_row_0/B:col10/1732653668542/Put/seqid=0 2024-11-26T20:41:09,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741985_1161 (size=12001) 2024-11-26T20:41:09,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/b62a41f4b859410f9e00c4425c735415 2024-11-26T20:41:09,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/4a58a81a2fee490a94d882ecb768f6a4 is 50, key is test_row_0/C:col10/1732653668542/Put/seqid=0 2024-11-26T20:41:09,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741986_1162 (size=12001) 2024-11-26T20:41:09,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/4a58a81a2fee490a94d882ecb768f6a4 2024-11-26T20:41:09,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T20:41:09,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:09,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:09,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:09,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:09,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:09,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:09,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/3591f216b9144464bdd9d738f3efa41f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3591f216b9144464bdd9d738f3efa41f 2024-11-26T20:41:09,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3591f216b9144464bdd9d738f3efa41f, entries=200, sequenceid=62, filesize=38.6 K 2024-11-26T20:41:09,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/b62a41f4b859410f9e00c4425c735415 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/b62a41f4b859410f9e00c4425c735415 2024-11-26T20:41:09,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/b62a41f4b859410f9e00c4425c735415, entries=150, sequenceid=62, filesize=11.7 K 2024-11-26T20:41:09,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/4a58a81a2fee490a94d882ecb768f6a4 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/4a58a81a2fee490a94d882ecb768f6a4 2024-11-26T20:41:09,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/4a58a81a2fee490a94d882ecb768f6a4, entries=150, sequenceid=62, filesize=11.7 K 2024-11-26T20:41:09,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 00a77bb01b2de4ac52368f2c3e602b3a in 943ms, sequenceid=62, compaction requested=true 2024-11-26T20:41:09,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:09,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a77bb01b2de4ac52368f2c3e602b3a:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:41:09,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:09,813 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:09,813 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:09,814 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:09,814 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:09,814 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 00a77bb01b2de4ac52368f2c3e602b3a/B is initiating minor compaction (all files) 2024-11-26T20:41:09,814 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 00a77bb01b2de4ac52368f2c3e602b3a/A is initiating minor compaction (all files) 2024-11-26T20:41:09,814 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a77bb01b2de4ac52368f2c3e602b3a/B in TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:09,814 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a77bb01b2de4ac52368f2c3e602b3a/A in TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:09,814 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/9f835546920f41cbb30df4a8d2781032, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/3d25af26302e4bc2a0a9848a21d72c91, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/b62a41f4b859410f9e00c4425c735415] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp, totalSize=35.2 K 2024-11-26T20:41:09,815 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3fd06b044e0d447a8f5b9ad7ebf667cc, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/a97b1c10e3ca4b5dbea59a05a71732a8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3591f216b9144464bdd9d738f3efa41f] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp, totalSize=107.5 K 2024-11-26T20:41:09,815 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f835546920f41cbb30df4a8d2781032, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732653667712 2024-11-26T20:41:09,815 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:09,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a77bb01b2de4ac52368f2c3e602b3a:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:41:09,815 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. files: [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3fd06b044e0d447a8f5b9ad7ebf667cc, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/a97b1c10e3ca4b5dbea59a05a71732a8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3591f216b9144464bdd9d738f3efa41f] 2024-11-26T20:41:09,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:09,815 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d25af26302e4bc2a0a9848a21d72c91, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732653667862 2024-11-26T20:41:09,816 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fd06b044e0d447a8f5b9ad7ebf667cc, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732653667709 2024-11-26T20:41:09,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a77bb01b2de4ac52368f2c3e602b3a:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:41:09,816 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting b62a41f4b859410f9e00c4425c735415, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732653668542 2024-11-26T20:41:09,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:09,816 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting a97b1c10e3ca4b5dbea59a05a71732a8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732653667862 2024-11-26T20:41:09,818 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3591f216b9144464bdd9d738f3efa41f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732653668539 2024-11-26T20:41:09,828 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a77bb01b2de4ac52368f2c3e602b3a#B#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:09,828 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/4f473b53d502433d9e04d8a005fe269c is 50, key is test_row_0/B:col10/1732653668542/Put/seqid=0 2024-11-26T20:41:09,829 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:09,838 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126934fd251771448d09f7124955b19b743_00a77bb01b2de4ac52368f2c3e602b3a store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:09,843 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126934fd251771448d09f7124955b19b743_00a77bb01b2de4ac52368f2c3e602b3a, store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:09,844 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126934fd251771448d09f7124955b19b743_00a77bb01b2de4ac52368f2c3e602b3a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:09,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741987_1163 (size=12104) 2024-11-26T20:41:09,862 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/4f473b53d502433d9e04d8a005fe269c as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/4f473b53d502433d9e04d8a005fe269c 2024-11-26T20:41:09,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741988_1164 (size=4469) 2024-11-26T20:41:09,864 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a77bb01b2de4ac52368f2c3e602b3a#A#compaction#145 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:09,865 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/824fd357f5594d99a3eaf72f2033bbb9 is 175, key is test_row_0/A:col10/1732653668542/Put/seqid=0 2024-11-26T20:41:09,870 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a77bb01b2de4ac52368f2c3e602b3a/B of 00a77bb01b2de4ac52368f2c3e602b3a into 4f473b53d502433d9e04d8a005fe269c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:09,871 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:09,871 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., storeName=00a77bb01b2de4ac52368f2c3e602b3a/B, priority=13, startTime=1732653669813; duration=0sec 2024-11-26T20:41:09,871 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:09,871 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a77bb01b2de4ac52368f2c3e602b3a:B 2024-11-26T20:41:09,871 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:09,872 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:09,872 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 00a77bb01b2de4ac52368f2c3e602b3a/C is initiating minor compaction (all files) 2024-11-26T20:41:09,872 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a77bb01b2de4ac52368f2c3e602b3a/C in TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:09,872 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/29448f06910c4989961d0edca282cb79, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/af7b01b219d24494a5d16dbb6c4d8a34, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/4a58a81a2fee490a94d882ecb768f6a4] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp, totalSize=35.2 K 2024-11-26T20:41:09,873 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 29448f06910c4989961d0edca282cb79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732653667712 2024-11-26T20:41:09,873 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting af7b01b219d24494a5d16dbb6c4d8a34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732653667862 2024-11-26T20:41:09,874 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a58a81a2fee490a94d882ecb768f6a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732653668542 2024-11-26T20:41:09,883 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a77bb01b2de4ac52368f2c3e602b3a#C#compaction#146 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:09,883 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/656257f9031645ab81e507f1835c2146 is 50, key is test_row_0/C:col10/1732653668542/Put/seqid=0 2024-11-26T20:41:09,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741989_1165 (size=31058) 2024-11-26T20:41:09,893 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/824fd357f5594d99a3eaf72f2033bbb9 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/824fd357f5594d99a3eaf72f2033bbb9 2024-11-26T20:41:09,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741990_1166 (size=12104) 2024-11-26T20:41:09,902 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a77bb01b2de4ac52368f2c3e602b3a/A of 00a77bb01b2de4ac52368f2c3e602b3a into 824fd357f5594d99a3eaf72f2033bbb9(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:09,902 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:09,902 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., storeName=00a77bb01b2de4ac52368f2c3e602b3a/A, priority=13, startTime=1732653669813; duration=0sec 2024-11-26T20:41:09,903 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:09,904 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a77bb01b2de4ac52368f2c3e602b3a:A 2024-11-26T20:41:09,909 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/656257f9031645ab81e507f1835c2146 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/656257f9031645ab81e507f1835c2146 2024-11-26T20:41:09,915 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a77bb01b2de4ac52368f2c3e602b3a/C of 00a77bb01b2de4ac52368f2c3e602b3a into 656257f9031645ab81e507f1835c2146(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:09,915 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:09,915 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., storeName=00a77bb01b2de4ac52368f2c3e602b3a/C, priority=13, startTime=1732653669816; duration=0sec 2024-11-26T20:41:09,915 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:09,915 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a77bb01b2de4ac52368f2c3e602b3a:C 2024-11-26T20:41:09,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T20:41:09,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:09,934 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 00a77bb01b2de4ac52368f2c3e602b3a 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-26T20:41:09,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=A 2024-11-26T20:41:09,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:09,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=B 2024-11-26T20:41:09,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:09,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=C 2024-11-26T20:41:09,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:09,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a28f2881646d43a888d7dad0186fbfb1_00a77bb01b2de4ac52368f2c3e602b3a is 50, key is test_row_0/A:col10/1732653669025/Put/seqid=0 2024-11-26T20:41:09,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741991_1167 (size=12154) 2024-11-26T20:41:09,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T20:41:10,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:10,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:10,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55170 deadline: 1732653730194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:10,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:10,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:10,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55242 deadline: 1732653730194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653730194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:10,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:10,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:10,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:10,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:10,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653730200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:10,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:10,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653730200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:10,226 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T20:41:10,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:10,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653730304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:10,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:10,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653730304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:10,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:10,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:10,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653730314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:10,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:10,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:10,382 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a28f2881646d43a888d7dad0186fbfb1_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a28f2881646d43a888d7dad0186fbfb1_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:10,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/1937d1de1b814aaeb842bf718d45927c, store: [table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:10,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/1937d1de1b814aaeb842bf718d45927c is 175, key is test_row_0/A:col10/1732653669025/Put/seqid=0 2024-11-26T20:41:10,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741992_1168 (size=30955) 2024-11-26T20:41:10,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653730514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:10,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:10,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653730514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:10,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:10,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:10,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653730524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:10,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:10,799 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/1937d1de1b814aaeb842bf718d45927c 2024-11-26T20:41:10,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/303fcfe6bbb547f184e8066e5363a5e1 is 50, key is test_row_0/B:col10/1732653669025/Put/seqid=0 2024-11-26T20:41:10,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653730824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:10,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:10,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:10,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653730827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:10,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:10,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653730844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:10,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:10,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:10,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741993_1169 (size=12001) 2024-11-26T20:41:10,863 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/303fcfe6bbb547f184e8066e5363a5e1 2024-11-26T20:41:10,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/817f9dd8f89f4707a228fae7c1d22879 is 50, key is test_row_0/C:col10/1732653669025/Put/seqid=0 2024-11-26T20:41:10,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741994_1170 (size=12001) 2024-11-26T20:41:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T20:41:11,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55170 deadline: 1732653731204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:11,212 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 00a77bb01b2de4ac52368f2c3e602b3a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:41:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:11,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55242 deadline: 1732653731214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:11,221 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 00a77bb01b2de4ac52368f2c3e602b3a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:41:11,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:11,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:11,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:11,313 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/817f9dd8f89f4707a228fae7c1d22879 2024-11-26T20:41:11,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/1937d1de1b814aaeb842bf718d45927c as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/1937d1de1b814aaeb842bf718d45927c 2024-11-26T20:41:11,326 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/1937d1de1b814aaeb842bf718d45927c, entries=150, sequenceid=99, filesize=30.2 K 2024-11-26T20:41:11,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/303fcfe6bbb547f184e8066e5363a5e1 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/303fcfe6bbb547f184e8066e5363a5e1 2024-11-26T20:41:11,343 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/303fcfe6bbb547f184e8066e5363a5e1, entries=150, sequenceid=99, filesize=11.7 K 2024-11-26T20:41:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/817f9dd8f89f4707a228fae7c1d22879 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/817f9dd8f89f4707a228fae7c1d22879 2024-11-26T20:41:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,350 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/817f9dd8f89f4707a228fae7c1d22879, entries=150, sequenceid=99, filesize=11.7 K 2024-11-26T20:41:11,351 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=67.09 KB/68700 for 00a77bb01b2de4ac52368f2c3e602b3a in 1417ms, sequenceid=99, compaction requested=false 2024-11-26T20:41:11,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:11,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:11,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-26T20:41:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-26T20:41:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:11,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-26T20:41:11,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5040 sec 2024-11-26T20:41:11,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00a77bb01b2de4ac52368f2c3e602b3a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T20:41:11,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=A 2024-11-26T20:41:11,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:11,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=B 2024-11-26T20:41:11,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:11,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=C 2024-11-26T20:41:11,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,362 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.5120 sec 2024-11-26T20:41:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126340ab04c1b1c49a4af8e5691439ea8c5_00a77bb01b2de4ac52368f2c3e602b3a is 50, key is test_row_0/A:col10/1732653670197/Put/seqid=0 2024-11-26T20:41:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741995_1171 (size=12154) 2024-11-26T20:41:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653731425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:11,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:11,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:11,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653731442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:11,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:11,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653731448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:11,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:11,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:11,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:11,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:11,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:11,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:11,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653731554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:11,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:11,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:11,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:11,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653731564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:11,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:11,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653731564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:11,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:11,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:11,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:11,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:11,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:11,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:11,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:11,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653731764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:11,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653731774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:11,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653731774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:11,839 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:11,847 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126340ab04c1b1c49a4af8e5691439ea8c5_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126340ab04c1b1c49a4af8e5691439ea8c5_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:11,848 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/3efa590d90c64296813b4074e43f74e7, store: [table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:11,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/3efa590d90c64296813b4074e43f74e7 is 175, key is test_row_0/A:col10/1732653670197/Put/seqid=0 2024-11-26T20:41:11,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741996_1172 (size=30955) 2024-11-26T20:41:12,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653732074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:12,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:12,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653732084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:12,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653732084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:12,287 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=113, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/3efa590d90c64296813b4074e43f74e7 2024-11-26T20:41:12,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/a5cf713272c74b578501535eabdd4745 is 50, key is test_row_0/B:col10/1732653670197/Put/seqid=0 2024-11-26T20:41:12,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741997_1173 (size=12001) 2024-11-26T20:41:12,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/a5cf713272c74b578501535eabdd4745 2024-11-26T20:41:12,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/3a46978eb63c4284a1e0e0a60f108b98 is 50, key is test_row_0/C:col10/1732653670197/Put/seqid=0 2024-11-26T20:41:12,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741998_1174 (size=12001) 2024-11-26T20:41:12,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/3a46978eb63c4284a1e0e0a60f108b98 2024-11-26T20:41:12,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/3efa590d90c64296813b4074e43f74e7 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3efa590d90c64296813b4074e43f74e7 2024-11-26T20:41:12,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3efa590d90c64296813b4074e43f74e7, entries=150, sequenceid=113, filesize=30.2 K 2024-11-26T20:41:12,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/a5cf713272c74b578501535eabdd4745 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/a5cf713272c74b578501535eabdd4745 2024-11-26T20:41:12,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/a5cf713272c74b578501535eabdd4745, entries=150, sequenceid=113, filesize=11.7 K 2024-11-26T20:41:12,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/3a46978eb63c4284a1e0e0a60f108b98 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/3a46978eb63c4284a1e0e0a60f108b98 2024-11-26T20:41:12,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/3a46978eb63c4284a1e0e0a60f108b98, entries=150, sequenceid=113, filesize=11.7 K 2024-11-26T20:41:12,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 00a77bb01b2de4ac52368f2c3e602b3a in 1082ms, sequenceid=113, compaction requested=true 2024-11-26T20:41:12,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:12,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a77bb01b2de4ac52368f2c3e602b3a:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:41:12,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:12,441 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:12,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a77bb01b2de4ac52368f2c3e602b3a:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:41:12,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:12,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a77bb01b2de4ac52368f2c3e602b3a:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:41:12,441 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:12,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:12,443 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:12,443 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:12,443 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 00a77bb01b2de4ac52368f2c3e602b3a/A is initiating minor compaction (all files) 2024-11-26T20:41:12,443 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 00a77bb01b2de4ac52368f2c3e602b3a/B is initiating minor compaction (all files) 2024-11-26T20:41:12,443 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a77bb01b2de4ac52368f2c3e602b3a/A in TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:12,443 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a77bb01b2de4ac52368f2c3e602b3a/B in TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:12,443 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/4f473b53d502433d9e04d8a005fe269c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/303fcfe6bbb547f184e8066e5363a5e1, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/a5cf713272c74b578501535eabdd4745] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp, totalSize=35.3 K 2024-11-26T20:41:12,443 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/824fd357f5594d99a3eaf72f2033bbb9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/1937d1de1b814aaeb842bf718d45927c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3efa590d90c64296813b4074e43f74e7] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp, totalSize=90.8 K 2024-11-26T20:41:12,443 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:12,443 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. files: [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/824fd357f5594d99a3eaf72f2033bbb9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/1937d1de1b814aaeb842bf718d45927c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3efa590d90c64296813b4074e43f74e7] 2024-11-26T20:41:12,444 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f473b53d502433d9e04d8a005fe269c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732653668542 2024-11-26T20:41:12,444 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 824fd357f5594d99a3eaf72f2033bbb9, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732653668542 2024-11-26T20:41:12,444 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 303fcfe6bbb547f184e8066e5363a5e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732653669017 2024-11-26T20:41:12,445 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1937d1de1b814aaeb842bf718d45927c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732653669017 2024-11-26T20:41:12,445 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting a5cf713272c74b578501535eabdd4745, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732653670197 2024-11-26T20:41:12,445 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3efa590d90c64296813b4074e43f74e7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732653670197 2024-11-26T20:41:12,457 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:12,461 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a77bb01b2de4ac52368f2c3e602b3a#B#compaction#154 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:12,461 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/3d72ee9775054dbe906fa6c7617c7af9 is 50, key is test_row_0/B:col10/1732653670197/Put/seqid=0 2024-11-26T20:41:12,473 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126814ff755666f4ea7bea41627b00df57f_00a77bb01b2de4ac52368f2c3e602b3a store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:12,476 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126814ff755666f4ea7bea41627b00df57f_00a77bb01b2de4ac52368f2c3e602b3a, store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:12,476 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126814ff755666f4ea7bea41627b00df57f_00a77bb01b2de4ac52368f2c3e602b3a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:12,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741999_1175 (size=12207) 2024-11-26T20:41:12,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742000_1176 (size=4469) 2024-11-26T20:41:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:12,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00a77bb01b2de4ac52368f2c3e602b3a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T20:41:12,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=A 2024-11-26T20:41:12,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:12,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=B 2024-11-26T20:41:12,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:12,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=C 2024-11-26T20:41:12,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:12,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263a6a61e5b10a438ea8a13fcea5f4fae4_00a77bb01b2de4ac52368f2c3e602b3a is 50, key is test_row_0/A:col10/1732653671443/Put/seqid=0 2024-11-26T20:41:12,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653732622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:12,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742001_1177 (size=14694) 2024-11-26T20:41:12,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:12,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653732624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:12,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653732626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:12,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:12,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:12,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653732734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:12,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653732734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:12,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653732734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:12,918 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/3d72ee9775054dbe906fa6c7617c7af9 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/3d72ee9775054dbe906fa6c7617c7af9 2024-11-26T20:41:12,920 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a77bb01b2de4ac52368f2c3e602b3a#A#compaction#153 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:12,921 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/6c642fd05ec3485e9dee65a14a43b930 is 175, key is test_row_0/A:col10/1732653670197/Put/seqid=0 2024-11-26T20:41:12,926 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a77bb01b2de4ac52368f2c3e602b3a/B of 00a77bb01b2de4ac52368f2c3e602b3a into 3d72ee9775054dbe906fa6c7617c7af9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:12,926 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:12,926 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., storeName=00a77bb01b2de4ac52368f2c3e602b3a/B, priority=13, startTime=1732653672441; duration=0sec 2024-11-26T20:41:12,926 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:12,926 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a77bb01b2de4ac52368f2c3e602b3a:B 2024-11-26T20:41:12,926 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:12,928 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:12,928 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 00a77bb01b2de4ac52368f2c3e602b3a/C is initiating minor compaction (all files) 2024-11-26T20:41:12,928 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a77bb01b2de4ac52368f2c3e602b3a/C in TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:12,928 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/656257f9031645ab81e507f1835c2146, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/817f9dd8f89f4707a228fae7c1d22879, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/3a46978eb63c4284a1e0e0a60f108b98] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp, totalSize=35.3 K 2024-11-26T20:41:12,929 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 656257f9031645ab81e507f1835c2146, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732653668542 2024-11-26T20:41:12,931 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 817f9dd8f89f4707a228fae7c1d22879, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732653669017 2024-11-26T20:41:12,932 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a46978eb63c4284a1e0e0a60f108b98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732653670197 2024-11-26T20:41:12,954 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a77bb01b2de4ac52368f2c3e602b3a#C#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:12,954 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/ed4329e2bdc044cf99d0c5caa8d60cf1 is 50, key is test_row_0/C:col10/1732653670197/Put/seqid=0 2024-11-26T20:41:12,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:12,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653732944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:12,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:12,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742002_1178 (size=31161) 2024-11-26T20:41:12,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653732944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:12,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:12,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:12,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653732944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:12,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:12,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:12,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:12,965 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/6c642fd05ec3485e9dee65a14a43b930 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/6c642fd05ec3485e9dee65a14a43b930 2024-11-26T20:41:12,971 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a77bb01b2de4ac52368f2c3e602b3a/A of 00a77bb01b2de4ac52368f2c3e602b3a into 6c642fd05ec3485e9dee65a14a43b930(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:12,971 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:12,971 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., storeName=00a77bb01b2de4ac52368f2c3e602b3a/A, priority=13, startTime=1732653672441; duration=0sec 2024-11-26T20:41:12,971 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:12,972 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a77bb01b2de4ac52368f2c3e602b3a:A 2024-11-26T20:41:12,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742003_1179 (size=12207) 2024-11-26T20:41:12,992 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/ed4329e2bdc044cf99d0c5caa8d60cf1 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/ed4329e2bdc044cf99d0c5caa8d60cf1 2024-11-26T20:41:12,998 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a77bb01b2de4ac52368f2c3e602b3a/C of 00a77bb01b2de4ac52368f2c3e602b3a into ed4329e2bdc044cf99d0c5caa8d60cf1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:12,998 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:12,999 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., storeName=00a77bb01b2de4ac52368f2c3e602b3a/C, priority=13, startTime=1732653672441; duration=0sec 2024-11-26T20:41:12,999 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:12,999 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a77bb01b2de4ac52368f2c3e602b3a:C 2024-11-26T20:41:13,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T20:41:13,025 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:13,026 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:41:13,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-26T20:41:13,028 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:41:13,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T20:41:13,030 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:13,031 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:41:13,031 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:41:13,038 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263a6a61e5b10a438ea8a13fcea5f4fae4_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263a6a61e5b10a438ea8a13fcea5f4fae4_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:13,040 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/9b86f9c2e6d2490ea9b75a59314f0e23, store: [table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:13,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/9b86f9c2e6d2490ea9b75a59314f0e23 is 175, key is test_row_0/A:col10/1732653671443/Put/seqid=0 2024-11-26T20:41:13,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742004_1180 (size=39649) 2024-11-26T20:41:13,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T20:41:13,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T20:41:13,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:13,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:13,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:13,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:13,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:13,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:13,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:13,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55170 deadline: 1732653733224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:13,229 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 00a77bb01b2de4ac52368f2c3e602b3a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:41:13,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:13,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:13,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:13,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:13,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55242 deadline: 1732653733244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:13,247 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 00a77bb01b2de4ac52368f2c3e602b3a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:41:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:13,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653733264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:13,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:13,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653733265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:13,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:13,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653733265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:13,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T20:41:13,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:13,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:13,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:13,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:13,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:13,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T20:41:13,457 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/9b86f9c2e6d2490ea9b75a59314f0e23 2024-11-26T20:41:13,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/4c678b100a444246bdde0d4fafefde6d is 50, key is test_row_0/B:col10/1732653671443/Put/seqid=0 2024-11-26T20:41:13,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T20:41:13,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:13,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:13,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:13,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:13,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:13,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742005_1181 (size=12101) 2024-11-26T20:41:13,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/4c678b100a444246bdde0d4fafefde6d 2024-11-26T20:41:13,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/047e2a23b7224ffd92f737ff2c4982dd is 50, key is test_row_0/C:col10/1732653671443/Put/seqid=0 2024-11-26T20:41:13,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742006_1182 (size=12101) 2024-11-26T20:41:13,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T20:41:13,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:13,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:13,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:13,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:13,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:13,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T20:41:13,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:13,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653733774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:13,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:13,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653733774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:13,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:13,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653733774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:13,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:13,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:13,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T20:41:13,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:13,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:13,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:13,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:13,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:13,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:13,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T20:41:13,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:13,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/047e2a23b7224ffd92f737ff2c4982dd 2024-11-26T20:41:13,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:13,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:13,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:13,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/9b86f9c2e6d2490ea9b75a59314f0e23 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/9b86f9c2e6d2490ea9b75a59314f0e23 2024-11-26T20:41:13,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/9b86f9c2e6d2490ea9b75a59314f0e23, entries=200, sequenceid=132, filesize=38.7 K 2024-11-26T20:41:13,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/4c678b100a444246bdde0d4fafefde6d as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/4c678b100a444246bdde0d4fafefde6d 2024-11-26T20:41:13,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/4c678b100a444246bdde0d4fafefde6d, entries=150, sequenceid=132, filesize=11.8 K 2024-11-26T20:41:13,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/047e2a23b7224ffd92f737ff2c4982dd as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/047e2a23b7224ffd92f737ff2c4982dd 2024-11-26T20:41:13,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/047e2a23b7224ffd92f737ff2c4982dd, entries=150, sequenceid=132, filesize=11.8 K 2024-11-26T20:41:13,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 00a77bb01b2de4ac52368f2c3e602b3a in 1399ms, sequenceid=132, compaction requested=false 2024-11-26T20:41:13,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:14,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T20:41:14,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:14,114 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 00a77bb01b2de4ac52368f2c3e602b3a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T20:41:14,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=A 2024-11-26T20:41:14,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:14,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=B 2024-11-26T20:41:14,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:14,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=C 2024-11-26T20:41:14,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:14,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c7faa3ffac894464b2986898a4d1112b_00a77bb01b2de4ac52368f2c3e602b3a is 50, key is test_row_0/A:col10/1732653672624/Put/seqid=0 2024-11-26T20:41:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T20:41:14,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742007_1183 (size=12304) 2024-11-26T20:41:14,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,196 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c7faa3ffac894464b2986898a4d1112b_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c7faa3ffac894464b2986898a4d1112b_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:14,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/ec896b442e4a4697b0b0490ef44dda35, store: [table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:14,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/ec896b442e4a4697b0b0490ef44dda35 is 175, key is test_row_0/A:col10/1732653672624/Put/seqid=0 2024-11-26T20:41:14,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742008_1184 (size=31105) 2024-11-26T20:41:14,214 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=148, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/ec896b442e4a4697b0b0490ef44dda35 2024-11-26T20:41:14,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/982eebfddbdb4028932326c5da241b78 is 50, key is test_row_0/B:col10/1732653672624/Put/seqid=0 2024-11-26T20:41:14,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742009_1185 (size=12151) 2024-11-26T20:41:14,243 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/982eebfddbdb4028932326c5da241b78 2024-11-26T20:41:14,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/d0efaf9c27494c999e735faa40c07ab9 is 50, key is test_row_0/C:col10/1732653672624/Put/seqid=0 2024-11-26T20:41:14,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742010_1186 (size=12151) 2024-11-26T20:41:14,278 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/d0efaf9c27494c999e735faa40c07ab9 2024-11-26T20:41:14,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/ec896b442e4a4697b0b0490ef44dda35 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/ec896b442e4a4697b0b0490ef44dda35 2024-11-26T20:41:14,290 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/ec896b442e4a4697b0b0490ef44dda35, entries=150, sequenceid=148, filesize=30.4 K 2024-11-26T20:41:14,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/982eebfddbdb4028932326c5da241b78 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/982eebfddbdb4028932326c5da241b78 2024-11-26T20:41:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,296 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/982eebfddbdb4028932326c5da241b78, entries=150, sequenceid=148, filesize=11.9 K 2024-11-26T20:41:14,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/d0efaf9c27494c999e735faa40c07ab9 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/d0efaf9c27494c999e735faa40c07ab9 2024-11-26T20:41:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,320 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/d0efaf9c27494c999e735faa40c07ab9, entries=150, sequenceid=148, filesize=11.9 K 2024-11-26T20:41:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,324 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 00a77bb01b2de4ac52368f2c3e602b3a in 210ms, sequenceid=148, compaction requested=true 2024-11-26T20:41:14,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:14,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:14,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-26T20:41:14,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-26T20:41:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-26T20:41:14,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2950 sec 2024-11-26T20:41:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,330 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.3020 sec 2024-11-26T20:41:14,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00a77bb01b2de4ac52368f2c3e602b3a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T20:41:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=A 2024-11-26T20:41:14,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:14,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=B 2024-11-26T20:41:14,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:14,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=C 2024-11-26T20:41:14,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112650864314e1ad466a9b3b9276e19a28a1_00a77bb01b2de4ac52368f2c3e602b3a is 50, key is test_row_0/A:col10/1732653674896/Put/seqid=0 2024-11-26T20:41:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742011_1187 (size=22268) 2024-11-26T20:41:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:14,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653674962, max timestamp=1732653674978Num uniques 859; 2024-11-26T20:41:14,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653674962, max timestamp=1732653674978Num uniques 859; 2024-11-26T20:41:14,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:41:14,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:41:14,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653674962, max timestamp=1732653674978Num uniques 859; 2024-11-26T20:41:14,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:41:15,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653735018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:15,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:15,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:15,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653735023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:15,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653735024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:15,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:15,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653735134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:15,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:15,146 INFO [master/cd6be161543e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-26T20:41:15,146 INFO [master/cd6be161543e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-26T20:41:15,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:15,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653735144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653735144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:15,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T20:41:15,175 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:15,176 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:41:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-26T20:41:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T20:41:15,179 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:41:15,180 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:41:15,180 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:41:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T20:41:15,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T20:41:15,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:15,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:15,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:15,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:15,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:15,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:15,345 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:15,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653735344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:15,351 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112650864314e1ad466a9b3b9276e19a28a1_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112650864314e1ad466a9b3b9276e19a28a1_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:15,352 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/807ef4a93cdc4f6aa38f9fd9b0889e4b, store: [table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:15,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/807ef4a93cdc4f6aa38f9fd9b0889e4b is 175, key is test_row_0/A:col10/1732653674896/Put/seqid=0 2024-11-26T20:41:15,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742012_1188 (size=65673) 2024-11-26T20:41:15,368 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/807ef4a93cdc4f6aa38f9fd9b0889e4b 2024-11-26T20:41:15,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653735364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:15,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:15,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653735364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:15,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:15,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/2170500e763246e7a74fecb33e2802ef is 50, key is test_row_0/B:col10/1732653674896/Put/seqid=0 2024-11-26T20:41:15,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742013_1189 (size=12151) 2024-11-26T20:41:15,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/2170500e763246e7a74fecb33e2802ef 2024-11-26T20:41:15,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/4f9c81cc6eec463e9f7164e4ffadfd62 is 50, key is test_row_0/C:col10/1732653674896/Put/seqid=0 2024-11-26T20:41:15,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742014_1190 (size=12151) 2024-11-26T20:41:15,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/4f9c81cc6eec463e9f7164e4ffadfd62 2024-11-26T20:41:15,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/807ef4a93cdc4f6aa38f9fd9b0889e4b as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/807ef4a93cdc4f6aa38f9fd9b0889e4b 2024-11-26T20:41:15,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/807ef4a93cdc4f6aa38f9fd9b0889e4b, entries=350, sequenceid=158, filesize=64.1 K 2024-11-26T20:41:15,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/2170500e763246e7a74fecb33e2802ef as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/2170500e763246e7a74fecb33e2802ef 2024-11-26T20:41:15,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/2170500e763246e7a74fecb33e2802ef, entries=150, sequenceid=158, filesize=11.9 K 2024-11-26T20:41:15,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/4f9c81cc6eec463e9f7164e4ffadfd62 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/4f9c81cc6eec463e9f7164e4ffadfd62 2024-11-26T20:41:15,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/4f9c81cc6eec463e9f7164e4ffadfd62, entries=150, sequenceid=158, filesize=11.9 K 2024-11-26T20:41:15,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 00a77bb01b2de4ac52368f2c3e602b3a in 556ms, sequenceid=158, compaction requested=true 2024-11-26T20:41:15,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:15,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a77bb01b2de4ac52368f2c3e602b3a:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:41:15,465 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:41:15,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:15,465 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:41:15,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a77bb01b2de4ac52368f2c3e602b3a:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:41:15,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:15,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a77bb01b2de4ac52368f2c3e602b3a:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:41:15,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:15,467 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48610 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:41:15,467 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 167588 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:41:15,467 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 00a77bb01b2de4ac52368f2c3e602b3a/B is initiating minor compaction (all files) 2024-11-26T20:41:15,467 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 00a77bb01b2de4ac52368f2c3e602b3a/A is initiating minor compaction (all files) 2024-11-26T20:41:15,467 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a77bb01b2de4ac52368f2c3e602b3a/B in TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:15,467 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a77bb01b2de4ac52368f2c3e602b3a/A in TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:15,467 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/3d72ee9775054dbe906fa6c7617c7af9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/4c678b100a444246bdde0d4fafefde6d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/982eebfddbdb4028932326c5da241b78, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/2170500e763246e7a74fecb33e2802ef] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp, totalSize=47.5 K 2024-11-26T20:41:15,467 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/6c642fd05ec3485e9dee65a14a43b930, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/9b86f9c2e6d2490ea9b75a59314f0e23, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/ec896b442e4a4697b0b0490ef44dda35, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/807ef4a93cdc4f6aa38f9fd9b0889e4b] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp, totalSize=163.7 K 2024-11-26T20:41:15,467 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:15,467 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. files: [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/6c642fd05ec3485e9dee65a14a43b930, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/9b86f9c2e6d2490ea9b75a59314f0e23, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/ec896b442e4a4697b0b0490ef44dda35, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/807ef4a93cdc4f6aa38f9fd9b0889e4b] 2024-11-26T20:41:15,467 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d72ee9775054dbe906fa6c7617c7af9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732653670197 2024-11-26T20:41:15,468 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c642fd05ec3485e9dee65a14a43b930, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732653670197 2024-11-26T20:41:15,468 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c678b100a444246bdde0d4fafefde6d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732653671420 2024-11-26T20:41:15,468 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b86f9c2e6d2490ea9b75a59314f0e23, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732653671420 2024-11-26T20:41:15,468 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 982eebfddbdb4028932326c5da241b78, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732653672620 2024-11-26T20:41:15,469 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec896b442e4a4697b0b0490ef44dda35, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732653672620 2024-11-26T20:41:15,469 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 2170500e763246e7a74fecb33e2802ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732653674896 2024-11-26T20:41:15,469 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 807ef4a93cdc4f6aa38f9fd9b0889e4b, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732653674886 2024-11-26T20:41:15,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T20:41:15,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:15,487 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 00a77bb01b2de4ac52368f2c3e602b3a 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-26T20:41:15,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=A 2024-11-26T20:41:15,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:15,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=B 2024-11-26T20:41:15,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:15,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=C 2024-11-26T20:41:15,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:15,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T20:41:15,504 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a77bb01b2de4ac52368f2c3e602b3a#B#compaction#165 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:15,504 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/85910462b30f41b3a1109b80fab70c6d is 50, key is test_row_0/B:col10/1732653674896/Put/seqid=0 2024-11-26T20:41:15,505 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:15,534 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112625ac9deaa6494c309102d3fcdf5a0018_00a77bb01b2de4ac52368f2c3e602b3a store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:15,537 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112625ac9deaa6494c309102d3fcdf5a0018_00a77bb01b2de4ac52368f2c3e602b3a, store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:15,537 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112625ac9deaa6494c309102d3fcdf5a0018_00a77bb01b2de4ac52368f2c3e602b3a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:15,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ce91974721854229878054aa5fa76323_00a77bb01b2de4ac52368f2c3e602b3a is 50, key is test_row_0/A:col10/1732653675013/Put/seqid=0 2024-11-26T20:41:15,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742015_1191 (size=12493) 2024-11-26T20:41:15,560 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/85910462b30f41b3a1109b80fab70c6d as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/85910462b30f41b3a1109b80fab70c6d 2024-11-26T20:41:15,569 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00a77bb01b2de4ac52368f2c3e602b3a/B of 00a77bb01b2de4ac52368f2c3e602b3a into 85910462b30f41b3a1109b80fab70c6d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:15,569 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:15,569 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., storeName=00a77bb01b2de4ac52368f2c3e602b3a/B, priority=12, startTime=1732653675465; duration=0sec 2024-11-26T20:41:15,569 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:15,569 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a77bb01b2de4ac52368f2c3e602b3a:B 2024-11-26T20:41:15,569 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:41:15,571 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48610 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:41:15,571 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 00a77bb01b2de4ac52368f2c3e602b3a/C is initiating minor compaction (all files) 2024-11-26T20:41:15,572 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a77bb01b2de4ac52368f2c3e602b3a/C in TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:15,572 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/ed4329e2bdc044cf99d0c5caa8d60cf1, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/047e2a23b7224ffd92f737ff2c4982dd, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/d0efaf9c27494c999e735faa40c07ab9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/4f9c81cc6eec463e9f7164e4ffadfd62] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp, totalSize=47.5 K 2024-11-26T20:41:15,573 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting ed4329e2bdc044cf99d0c5caa8d60cf1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732653670197 2024-11-26T20:41:15,573 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 047e2a23b7224ffd92f737ff2c4982dd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732653671420 2024-11-26T20:41:15,574 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting d0efaf9c27494c999e735faa40c07ab9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732653672620 2024-11-26T20:41:15,574 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f9c81cc6eec463e9f7164e4ffadfd62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732653674896 2024-11-26T20:41:15,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742016_1192 (size=4469) 2024-11-26T20:41:15,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742017_1193 (size=12304) 2024-11-26T20:41:15,593 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a77bb01b2de4ac52368f2c3e602b3a#C#compaction#168 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:15,593 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/d6f53556cff64bbe81c72f9d6a29f926 is 50, key is test_row_0/C:col10/1732653674896/Put/seqid=0 2024-11-26T20:41:15,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742018_1194 (size=12493) 2024-11-26T20:41:15,626 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/d6f53556cff64bbe81c72f9d6a29f926 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/d6f53556cff64bbe81c72f9d6a29f926 2024-11-26T20:41:15,632 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00a77bb01b2de4ac52368f2c3e602b3a/C of 00a77bb01b2de4ac52368f2c3e602b3a into d6f53556cff64bbe81c72f9d6a29f926(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:15,632 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:15,632 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., storeName=00a77bb01b2de4ac52368f2c3e602b3a/C, priority=12, startTime=1732653675466; duration=0sec 2024-11-26T20:41:15,632 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:15,632 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a77bb01b2de4ac52368f2c3e602b3a:C 2024-11-26T20:41:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:15,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:15,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653735682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653735683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:15,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653735684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:15,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:15,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653735794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:15,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653735794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:15,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653735794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:15,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:15,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:15,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T20:41:15,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:15,989 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a77bb01b2de4ac52368f2c3e602b3a#A#compaction#166 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:15,990 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/48228e3ed9bf40f7bc925b54117a2f11 is 175, key is test_row_0/A:col10/1732653674896/Put/seqid=0 2024-11-26T20:41:15,994 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ce91974721854229878054aa5fa76323_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ce91974721854229878054aa5fa76323_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:15,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/655505aabdd94d41b3f47ac99f902cd1, store: [table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:15,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/655505aabdd94d41b3f47ac99f902cd1 is 175, key is test_row_0/A:col10/1732653675013/Put/seqid=0 2024-11-26T20:41:16,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653736003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:16,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653736004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:16,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:16,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653736004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:16,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742019_1195 (size=31447) 2024-11-26T20:41:16,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742020_1196 (size=31105) 2024-11-26T20:41:16,068 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/655505aabdd94d41b3f47ac99f902cd1 2024-11-26T20:41:16,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/83f3cf3f210041eda6e6f1910fb5fa39 is 50, key is test_row_0/B:col10/1732653675013/Put/seqid=0 2024-11-26T20:41:16,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742021_1197 (size=12151) 2024-11-26T20:41:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T20:41:16,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653736314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:16,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:16,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:16,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:16,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653736314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:16,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:16,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:16,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:16,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:16,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653736314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:16,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:16,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:16,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:16,463 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/48228e3ed9bf40f7bc925b54117a2f11 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/48228e3ed9bf40f7bc925b54117a2f11 2024-11-26T20:41:16,470 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00a77bb01b2de4ac52368f2c3e602b3a/A of 00a77bb01b2de4ac52368f2c3e602b3a into 48228e3ed9bf40f7bc925b54117a2f11(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-26T20:41:16,470 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:16,470 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., storeName=00a77bb01b2de4ac52368f2c3e602b3a/A, priority=12, startTime=1732653675465; duration=1sec 2024-11-26T20:41:16,470 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:16,470 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a77bb01b2de4ac52368f2c3e602b3a:A 2024-11-26T20:41:16,498 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/83f3cf3f210041eda6e6f1910fb5fa39 2024-11-26T20:41:16,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/f3a8ce8f70464258be5c985315292d45 is 50, key is test_row_0/C:col10/1732653675013/Put/seqid=0 2024-11-26T20:41:16,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742022_1198 (size=12151) 2024-11-26T20:41:16,535 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/f3a8ce8f70464258be5c985315292d45 2024-11-26T20:41:16,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/655505aabdd94d41b3f47ac99f902cd1 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/655505aabdd94d41b3f47ac99f902cd1 2024-11-26T20:41:16,550 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/655505aabdd94d41b3f47ac99f902cd1, entries=150, sequenceid=194, filesize=30.4 K 2024-11-26T20:41:16,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/83f3cf3f210041eda6e6f1910fb5fa39 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/83f3cf3f210041eda6e6f1910fb5fa39 2024-11-26T20:41:16,561 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/83f3cf3f210041eda6e6f1910fb5fa39, entries=150, sequenceid=194, filesize=11.9 K 2024-11-26T20:41:16,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/f3a8ce8f70464258be5c985315292d45 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/f3a8ce8f70464258be5c985315292d45 2024-11-26T20:41:16,569 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/f3a8ce8f70464258be5c985315292d45, entries=150, sequenceid=194, filesize=11.9 K 2024-11-26T20:41:16,570 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 00a77bb01b2de4ac52368f2c3e602b3a in 1083ms, sequenceid=194, compaction requested=false 2024-11-26T20:41:16,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:16,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:16,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-26T20:41:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-26T20:41:16,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-26T20:41:16,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3960 sec 2024-11-26T20:41:16,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.4030 sec 2024-11-26T20:41:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:16,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00a77bb01b2de4ac52368f2c3e602b3a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T20:41:16,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=A 2024-11-26T20:41:16,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:16,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=B 2024-11-26T20:41:16,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:16,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=C 2024-11-26T20:41:16,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:16,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f1212472e3d5470385e153a834db8d0f_00a77bb01b2de4ac52368f2c3e602b3a is 50, key is test_row_0/A:col10/1732653675662/Put/seqid=0 2024-11-26T20:41:16,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742023_1199 (size=14794) 2024-11-26T20:41:16,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732653676863, max timestamp=1732653676877Num uniques 859; 2024-11-26T20:41:16,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-26T20:41:16,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732653676863, max timestamp=1732653676877Num uniques 859; 2024-11-26T20:41:16,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-26T20:41:16,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732653676863, max timestamp=1732653676877Num uniques 859; 2024-11-26T20:41:16,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-26T20:41:16,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653736911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:16,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653736911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:16,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653736912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:17,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653737023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:17,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653737023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:17,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653737034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:17,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:17,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:17,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:17,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653737234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:17,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653737234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:17,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:17,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:17,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:17,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:17,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:17,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:17,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:17,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653737243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:17,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:17,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:17,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:17,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55170 deadline: 1732653737264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:17,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55242 deadline: 1732653737264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:17,265 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:17,265 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 00a77bb01b2de4ac52368f2c3e602b3a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:41:17,266 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:17,266 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:17,266 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:17,265 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 00a77bb01b2de4ac52368f2c3e602b3a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:41:17,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:17,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:17,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:17,269 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f1212472e3d5470385e153a834db8d0f_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f1212472e3d5470385e153a834db8d0f_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:17,270 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/a3710bb1d26f4e9485367fbf624a9936, store: [table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:17,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/a3710bb1d26f4e9485367fbf624a9936 is 175, key is test_row_0/A:col10/1732653675662/Put/seqid=0 2024-11-26T20:41:17,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742024_1200 (size=39749) 2024-11-26T20:41:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T20:41:17,334 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:17,335 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:41:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-26T20:41:17,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T20:41:17,337 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:41:17,339 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:41:17,339 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:41:17,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T20:41:17,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-26T20:41:17,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:17,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:17,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:17,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:17,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:17,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:17,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:17,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653737544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:17,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:17,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653737544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:17,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:17,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:17,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:17,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:17,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653737554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:17,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:17,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:17,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:17,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-26T20:41:17,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:17,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:17,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:17,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:17,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:17,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:17,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T20:41:17,680 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/a3710bb1d26f4e9485367fbf624a9936 2024-11-26T20:41:17,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/1e6ae85c9efd4fdca64d97d1221b6aae is 50, key is test_row_0/B:col10/1732653675662/Put/seqid=0 2024-11-26T20:41:17,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742025_1201 (size=12151) 2024-11-26T20:41:17,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-26T20:41:17,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:17,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:17,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:17,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:17,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:17,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-26T20:41:17,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:17,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:17,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:17,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:17,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T20:41:18,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653738054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:18,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:18,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:18,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653738054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:18,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:18,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:18,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653738063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:18,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:18,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:18,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:18,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/1e6ae85c9efd4fdca64d97d1221b6aae 2024-11-26T20:41:18,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/b170545f5c77496bbca658d6d9cc7fb6 is 50, key is test_row_0/C:col10/1732653675662/Put/seqid=0 2024-11-26T20:41:18,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-26T20:41:18,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:18,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:18,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:18,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:18,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:18,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742026_1202 (size=12151) 2024-11-26T20:41:18,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-26T20:41:18,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:18,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:18,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:18,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:18,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:18,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-26T20:41:18,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:18,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:18,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:18,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:18,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:18,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T20:41:18,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/b170545f5c77496bbca658d6d9cc7fb6 2024-11-26T20:41:18,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/a3710bb1d26f4e9485367fbf624a9936 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/a3710bb1d26f4e9485367fbf624a9936 2024-11-26T20:41:18,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/a3710bb1d26f4e9485367fbf624a9936, entries=200, sequenceid=208, filesize=38.8 K 2024-11-26T20:41:18,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/1e6ae85c9efd4fdca64d97d1221b6aae as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/1e6ae85c9efd4fdca64d97d1221b6aae 2024-11-26T20:41:18,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/1e6ae85c9efd4fdca64d97d1221b6aae, entries=150, sequenceid=208, filesize=11.9 K 2024-11-26T20:41:18,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/b170545f5c77496bbca658d6d9cc7fb6 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/b170545f5c77496bbca658d6d9cc7fb6 2024-11-26T20:41:18,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/b170545f5c77496bbca658d6d9cc7fb6, entries=150, sequenceid=208, filesize=11.9 K 2024-11-26T20:41:18,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 00a77bb01b2de4ac52368f2c3e602b3a in 1705ms, sequenceid=208, compaction requested=true 2024-11-26T20:41:18,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:18,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a77bb01b2de4ac52368f2c3e602b3a:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:41:18,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:18,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a77bb01b2de4ac52368f2c3e602b3a:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:41:18,531 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:18,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:18,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a77bb01b2de4ac52368f2c3e602b3a:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:41:18,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:18,531 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:18,532 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:18,532 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:18,533 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 00a77bb01b2de4ac52368f2c3e602b3a/A is initiating minor compaction (all files) 2024-11-26T20:41:18,533 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 00a77bb01b2de4ac52368f2c3e602b3a/B is initiating minor compaction (all files) 2024-11-26T20:41:18,533 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a77bb01b2de4ac52368f2c3e602b3a/A in TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:18,533 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a77bb01b2de4ac52368f2c3e602b3a/B in TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:18,533 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/48228e3ed9bf40f7bc925b54117a2f11, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/655505aabdd94d41b3f47ac99f902cd1, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/a3710bb1d26f4e9485367fbf624a9936] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp, totalSize=99.9 K 2024-11-26T20:41:18,533 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:18,533 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/85910462b30f41b3a1109b80fab70c6d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/83f3cf3f210041eda6e6f1910fb5fa39, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/1e6ae85c9efd4fdca64d97d1221b6aae] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp, totalSize=35.9 K 2024-11-26T20:41:18,533 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. files: [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/48228e3ed9bf40f7bc925b54117a2f11, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/655505aabdd94d41b3f47ac99f902cd1, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/a3710bb1d26f4e9485367fbf624a9936] 2024-11-26T20:41:18,533 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48228e3ed9bf40f7bc925b54117a2f11, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732653674896 2024-11-26T20:41:18,533 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 85910462b30f41b3a1109b80fab70c6d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732653674896 2024-11-26T20:41:18,533 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 655505aabdd94d41b3f47ac99f902cd1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732653674991 2024-11-26T20:41:18,533 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 83f3cf3f210041eda6e6f1910fb5fa39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732653674991 2024-11-26T20:41:18,534 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3710bb1d26f4e9485367fbf624a9936, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732653675662 2024-11-26T20:41:18,534 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e6ae85c9efd4fdca64d97d1221b6aae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732653675662 2024-11-26T20:41:18,540 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:18,543 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a77bb01b2de4ac52368f2c3e602b3a#B#compaction#175 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:18,543 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/fb3dc7075a614a71aea43cc836846b14 is 50, key is test_row_0/B:col10/1732653675662/Put/seqid=0 2024-11-26T20:41:18,546 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411268ea1d39f91f7484681292ab4f35b6652_00a77bb01b2de4ac52368f2c3e602b3a store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:18,548 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411268ea1d39f91f7484681292ab4f35b6652_00a77bb01b2de4ac52368f2c3e602b3a, store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:18,548 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411268ea1d39f91f7484681292ab4f35b6652_00a77bb01b2de4ac52368f2c3e602b3a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:18,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742027_1203 (size=12595) 2024-11-26T20:41:18,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742028_1204 (size=4469) 2024-11-26T20:41:18,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-26T20:41:18,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:18,566 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 00a77bb01b2de4ac52368f2c3e602b3a 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-26T20:41:18,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=A 2024-11-26T20:41:18,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:18,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=B 2024-11-26T20:41:18,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:18,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=C 2024-11-26T20:41:18,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:18,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126188182a9d0f64ce3bfac5beba8c87483_00a77bb01b2de4ac52368f2c3e602b3a is 50, key is test_row_0/A:col10/1732653676906/Put/seqid=0 2024-11-26T20:41:18,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742029_1205 (size=12304) 2024-11-26T20:41:18,955 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a77bb01b2de4ac52368f2c3e602b3a#A#compaction#174 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:18,955 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/df82671c56d540f2a59a9f9615220be2 is 175, key is test_row_0/A:col10/1732653675662/Put/seqid=0 2024-11-26T20:41:18,960 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/fb3dc7075a614a71aea43cc836846b14 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/fb3dc7075a614a71aea43cc836846b14 2024-11-26T20:41:18,966 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a77bb01b2de4ac52368f2c3e602b3a/B of 00a77bb01b2de4ac52368f2c3e602b3a into fb3dc7075a614a71aea43cc836846b14(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:18,966 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:18,966 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., storeName=00a77bb01b2de4ac52368f2c3e602b3a/B, priority=13, startTime=1732653678531; duration=0sec 2024-11-26T20:41:18,966 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:18,966 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a77bb01b2de4ac52368f2c3e602b3a:B 2024-11-26T20:41:18,966 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:18,967 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:18,967 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 00a77bb01b2de4ac52368f2c3e602b3a/C is initiating minor compaction (all files) 2024-11-26T20:41:18,967 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a77bb01b2de4ac52368f2c3e602b3a/C in TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:18,967 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/d6f53556cff64bbe81c72f9d6a29f926, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/f3a8ce8f70464258be5c985315292d45, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/b170545f5c77496bbca658d6d9cc7fb6] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp, totalSize=35.9 K 2024-11-26T20:41:18,968 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting d6f53556cff64bbe81c72f9d6a29f926, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732653674896 2024-11-26T20:41:18,968 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting f3a8ce8f70464258be5c985315292d45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732653674991 2024-11-26T20:41:18,968 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting b170545f5c77496bbca658d6d9cc7fb6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732653675662 2024-11-26T20:41:18,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742030_1206 (size=31549) 2024-11-26T20:41:18,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:18,987 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/df82671c56d540f2a59a9f9615220be2 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/df82671c56d540f2a59a9f9615220be2 2024-11-26T20:41:18,987 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a77bb01b2de4ac52368f2c3e602b3a#C#compaction#177 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:18,987 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/57ead70af2934c39aac9dc9ee0579ea9 is 50, key is test_row_0/C:col10/1732653675662/Put/seqid=0 2024-11-26T20:41:18,992 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126188182a9d0f64ce3bfac5beba8c87483_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126188182a9d0f64ce3bfac5beba8c87483_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:18,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/31fa73e13cb442318a59279a70b49c4f, store: [table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:18,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/31fa73e13cb442318a59279a70b49c4f is 175, key is test_row_0/A:col10/1732653676906/Put/seqid=0 2024-11-26T20:41:18,996 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a77bb01b2de4ac52368f2c3e602b3a/A of 00a77bb01b2de4ac52368f2c3e602b3a into df82671c56d540f2a59a9f9615220be2(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:18,996 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:18,996 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., storeName=00a77bb01b2de4ac52368f2c3e602b3a/A, priority=13, startTime=1732653678531; duration=0sec 2024-11-26T20:41:18,996 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:18,996 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a77bb01b2de4ac52368f2c3e602b3a:A 2024-11-26T20:41:19,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742031_1207 (size=12595) 2024-11-26T20:41:19,017 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/57ead70af2934c39aac9dc9ee0579ea9 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/57ead70af2934c39aac9dc9ee0579ea9 2024-11-26T20:41:19,024 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a77bb01b2de4ac52368f2c3e602b3a/C of 00a77bb01b2de4ac52368f2c3e602b3a into 57ead70af2934c39aac9dc9ee0579ea9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:19,024 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:19,024 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., storeName=00a77bb01b2de4ac52368f2c3e602b3a/C, priority=13, startTime=1732653678531; duration=0sec 2024-11-26T20:41:19,024 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:19,024 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a77bb01b2de4ac52368f2c3e602b3a:C 2024-11-26T20:41:19,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742032_1208 (size=31105) 2024-11-26T20:41:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:19,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:19,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653739089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:19,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653739090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:19,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:19,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653739091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:19,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:19,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:19,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653739203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:19,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:19,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653739203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:19,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:19,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:19,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653739204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:19,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:19,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:19,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653739414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:19,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653739414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:19,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653739415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:19,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:19,428 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/31fa73e13cb442318a59279a70b49c4f 2024-11-26T20:41:19,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/8f36750e3eb6403382cd77efdd4a5d8c is 50, key is test_row_0/B:col10/1732653676906/Put/seqid=0 2024-11-26T20:41:19,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742033_1209 (size=12151) 2024-11-26T20:41:19,442 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/8f36750e3eb6403382cd77efdd4a5d8c 2024-11-26T20:41:19,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/73406d71a1834e80adeb6a43d9752c06 is 50, key is test_row_0/C:col10/1732653676906/Put/seqid=0 2024-11-26T20:41:19,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742034_1210 (size=12151) 2024-11-26T20:41:19,459 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/73406d71a1834e80adeb6a43d9752c06 2024-11-26T20:41:19,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/31fa73e13cb442318a59279a70b49c4f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/31fa73e13cb442318a59279a70b49c4f 2024-11-26T20:41:19,475 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/31fa73e13cb442318a59279a70b49c4f, entries=150, sequenceid=242, filesize=30.4 K 2024-11-26T20:41:19,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/8f36750e3eb6403382cd77efdd4a5d8c as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/8f36750e3eb6403382cd77efdd4a5d8c 2024-11-26T20:41:19,483 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/8f36750e3eb6403382cd77efdd4a5d8c, entries=150, sequenceid=242, filesize=11.9 K 2024-11-26T20:41:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T20:41:19,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/73406d71a1834e80adeb6a43d9752c06 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/73406d71a1834e80adeb6a43d9752c06 2024-11-26T20:41:19,490 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/73406d71a1834e80adeb6a43d9752c06, entries=150, sequenceid=242, filesize=11.9 K 2024-11-26T20:41:19,491 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 00a77bb01b2de4ac52368f2c3e602b3a in 925ms, sequenceid=242, compaction requested=false 2024-11-26T20:41:19,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:19,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:19,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-26T20:41:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-26T20:41:19,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-26T20:41:19,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1540 sec 2024-11-26T20:41:19,497 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.1610 sec 2024-11-26T20:41:19,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:19,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00a77bb01b2de4ac52368f2c3e602b3a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T20:41:19,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=A 2024-11-26T20:41:19,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:19,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=B 2024-11-26T20:41:19,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:19,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=C 2024-11-26T20:41:19,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:19,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112684639e58491941d4828f353187fc3be8_00a77bb01b2de4ac52368f2c3e602b3a is 50, key is test_row_0/A:col10/1732653679726/Put/seqid=0 2024-11-26T20:41:19,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742035_1211 (size=19924) 2024-11-26T20:41:19,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:19,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653739757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:19,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:19,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:19,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653739760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:19,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:19,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653739763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:19,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:19,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:19,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653739874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:19,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:19,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653739874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:19,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:19,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653739874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:20,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653740084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:20,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653740084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:20,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653740084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:20,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:20,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:20,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:20,143 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:20,147 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112684639e58491941d4828f353187fc3be8_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112684639e58491941d4828f353187fc3be8_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:20,148 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/9a742c1f5b5a474f92826cfbe4286d31, store: [table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:20,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/9a742c1f5b5a474f92826cfbe4286d31 is 175, key is test_row_0/A:col10/1732653679726/Put/seqid=0 2024-11-26T20:41:20,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742036_1212 (size=57183) 2024-11-26T20:41:20,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653740393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:20,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653740393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:20,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653740394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:20,554 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/9a742c1f5b5a474f92826cfbe4286d31 2024-11-26T20:41:20,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/59d5d6a0b90c4819a2510539bf7ea69a is 50, key is test_row_0/B:col10/1732653679726/Put/seqid=0 2024-11-26T20:41:20,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742037_1213 (size=12251) 2024-11-26T20:41:20,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653740904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:20,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653740904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:20,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653740904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:20,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/59d5d6a0b90c4819a2510539bf7ea69a 2024-11-26T20:41:20,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/4767b7e18b994c57aa42d452b2e3a79b is 50, key is test_row_0/C:col10/1732653679726/Put/seqid=0 2024-11-26T20:41:20,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742038_1214 (size=12251) 2024-11-26T20:41:21,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/4767b7e18b994c57aa42d452b2e3a79b 2024-11-26T20:41:21,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/9a742c1f5b5a474f92826cfbe4286d31 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/9a742c1f5b5a474f92826cfbe4286d31 2024-11-26T20:41:21,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/9a742c1f5b5a474f92826cfbe4286d31, entries=300, sequenceid=259, filesize=55.8 K 2024-11-26T20:41:21,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/59d5d6a0b90c4819a2510539bf7ea69a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/59d5d6a0b90c4819a2510539bf7ea69a 2024-11-26T20:41:21,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/59d5d6a0b90c4819a2510539bf7ea69a, entries=150, sequenceid=259, filesize=12.0 K 2024-11-26T20:41:21,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/4767b7e18b994c57aa42d452b2e3a79b as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/4767b7e18b994c57aa42d452b2e3a79b 2024-11-26T20:41:21,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/4767b7e18b994c57aa42d452b2e3a79b, entries=150, sequenceid=259, filesize=12.0 K 2024-11-26T20:41:21,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 00a77bb01b2de4ac52368f2c3e602b3a in 1684ms, sequenceid=259, compaction requested=true 2024-11-26T20:41:21,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:21,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a77bb01b2de4ac52368f2c3e602b3a:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:41:21,410 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:21,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:21,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a77bb01b2de4ac52368f2c3e602b3a:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:41:21,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:21,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a77bb01b2de4ac52368f2c3e602b3a:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:41:21,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:21,410 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:21,411 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119837 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:21,411 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36997 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:21,411 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 00a77bb01b2de4ac52368f2c3e602b3a/A is initiating minor compaction (all files) 2024-11-26T20:41:21,411 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 00a77bb01b2de4ac52368f2c3e602b3a/B is initiating minor compaction (all files) 2024-11-26T20:41:21,411 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a77bb01b2de4ac52368f2c3e602b3a/A in TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:21,411 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a77bb01b2de4ac52368f2c3e602b3a/B in TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:21,411 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/df82671c56d540f2a59a9f9615220be2, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/31fa73e13cb442318a59279a70b49c4f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/9a742c1f5b5a474f92826cfbe4286d31] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp, totalSize=117.0 K 2024-11-26T20:41:21,411 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:21,411 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/fb3dc7075a614a71aea43cc836846b14, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/8f36750e3eb6403382cd77efdd4a5d8c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/59d5d6a0b90c4819a2510539bf7ea69a] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp, totalSize=36.1 K 2024-11-26T20:41:21,411 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. files: [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/df82671c56d540f2a59a9f9615220be2, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/31fa73e13cb442318a59279a70b49c4f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/9a742c1f5b5a474f92826cfbe4286d31] 2024-11-26T20:41:21,412 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb3dc7075a614a71aea43cc836846b14, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732653675662 2024-11-26T20:41:21,412 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting df82671c56d540f2a59a9f9615220be2, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732653675662 2024-11-26T20:41:21,412 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f36750e3eb6403382cd77efdd4a5d8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732653676906 2024-11-26T20:41:21,412 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 31fa73e13cb442318a59279a70b49c4f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732653676906 2024-11-26T20:41:21,412 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59d5d6a0b90c4819a2510539bf7ea69a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732653679090 2024-11-26T20:41:21,412 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a742c1f5b5a474f92826cfbe4286d31, keycount=300, bloomtype=ROW, size=55.8 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732653679086 2024-11-26T20:41:21,419 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:21,420 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a77bb01b2de4ac52368f2c3e602b3a#B#compaction#184 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:21,421 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/1c983009ba7948c1a4a5645a186cbeac is 50, key is test_row_0/B:col10/1732653679726/Put/seqid=0 2024-11-26T20:41:21,423 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126bafb752732d642b19400b7a9d1d025fa_00a77bb01b2de4ac52368f2c3e602b3a store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:21,425 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126bafb752732d642b19400b7a9d1d025fa_00a77bb01b2de4ac52368f2c3e602b3a, store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:21,425 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126bafb752732d642b19400b7a9d1d025fa_00a77bb01b2de4ac52368f2c3e602b3a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:21,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742040_1216 (size=4469) 2024-11-26T20:41:21,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742039_1215 (size=12797) 2024-11-26T20:41:21,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T20:41:21,504 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:21,504 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:41:21,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-26T20:41:21,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T20:41:21,506 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:41:21,507 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:41:21,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:41:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T20:41:21,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-26T20:41:21,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:21,659 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 00a77bb01b2de4ac52368f2c3e602b3a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T20:41:21,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=A 2024-11-26T20:41:21,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:21,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=B 2024-11-26T20:41:21,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:21,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=C 2024-11-26T20:41:21,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:21,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126fe76645d76ed4911aa7b1fa900f65aac_00a77bb01b2de4ac52368f2c3e602b3a is 50, key is test_row_0/A:col10/1732653679758/Put/seqid=0 2024-11-26T20:41:21,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742041_1217 (size=12454) 2024-11-26T20:41:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T20:41:21,838 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a77bb01b2de4ac52368f2c3e602b3a#A#compaction#183 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:21,838 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/385f69cfa07b47f4b533cdc2c48b257b is 175, key is test_row_0/A:col10/1732653679726/Put/seqid=0 2024-11-26T20:41:21,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742042_1218 (size=31751) 2024-11-26T20:41:21,846 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/1c983009ba7948c1a4a5645a186cbeac as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/1c983009ba7948c1a4a5645a186cbeac 2024-11-26T20:41:21,851 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/385f69cfa07b47f4b533cdc2c48b257b as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/385f69cfa07b47f4b533cdc2c48b257b 2024-11-26T20:41:21,853 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a77bb01b2de4ac52368f2c3e602b3a/B of 00a77bb01b2de4ac52368f2c3e602b3a into 1c983009ba7948c1a4a5645a186cbeac(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:21,853 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:21,853 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., storeName=00a77bb01b2de4ac52368f2c3e602b3a/B, priority=13, startTime=1732653681410; duration=0sec 2024-11-26T20:41:21,853 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:21,853 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a77bb01b2de4ac52368f2c3e602b3a:B 2024-11-26T20:41:21,853 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:21,858 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36997 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:21,858 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 00a77bb01b2de4ac52368f2c3e602b3a/C is initiating minor compaction (all files) 2024-11-26T20:41:21,858 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a77bb01b2de4ac52368f2c3e602b3a/C in TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:21,858 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/57ead70af2934c39aac9dc9ee0579ea9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/73406d71a1834e80adeb6a43d9752c06, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/4767b7e18b994c57aa42d452b2e3a79b] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp, totalSize=36.1 K 2024-11-26T20:41:21,859 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57ead70af2934c39aac9dc9ee0579ea9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732653675662 2024-11-26T20:41:21,859 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73406d71a1834e80adeb6a43d9752c06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732653676906 2024-11-26T20:41:21,860 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4767b7e18b994c57aa42d452b2e3a79b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732653679090 2024-11-26T20:41:21,864 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a77bb01b2de4ac52368f2c3e602b3a/A of 00a77bb01b2de4ac52368f2c3e602b3a into 385f69cfa07b47f4b533cdc2c48b257b(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:21,864 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:21,864 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., storeName=00a77bb01b2de4ac52368f2c3e602b3a/A, priority=13, startTime=1732653681410; duration=0sec 2024-11-26T20:41:21,864 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:21,864 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a77bb01b2de4ac52368f2c3e602b3a:A 2024-11-26T20:41:21,871 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a77bb01b2de4ac52368f2c3e602b3a#C#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:21,872 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/74965d5d48494d5a8f496b6c63dc669f is 50, key is test_row_0/C:col10/1732653679726/Put/seqid=0 2024-11-26T20:41:21,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742043_1219 (size=12797) 2024-11-26T20:41:21,890 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/74965d5d48494d5a8f496b6c63dc669f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/74965d5d48494d5a8f496b6c63dc669f 2024-11-26T20:41:21,896 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00a77bb01b2de4ac52368f2c3e602b3a/C of 00a77bb01b2de4ac52368f2c3e602b3a into 74965d5d48494d5a8f496b6c63dc669f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:21,896 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:21,896 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., storeName=00a77bb01b2de4ac52368f2c3e602b3a/C, priority=13, startTime=1732653681410; duration=0sec 2024-11-26T20:41:21,896 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:21,896 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a77bb01b2de4ac52368f2c3e602b3a:C 2024-11-26T20:41:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:21,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:21,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653741935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:21,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653741935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:21,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653741937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:21,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:21,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:21,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:22,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653742044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:22,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:22,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653742044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653742044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:22,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:22,076 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126fe76645d76ed4911aa7b1fa900f65aac_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126fe76645d76ed4911aa7b1fa900f65aac_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:22,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/1c593224e3b14492885899bb10d09567, store: [table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:22,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/1c593224e3b14492885899bb10d09567 is 175, key is test_row_0/A:col10/1732653679758/Put/seqid=0 2024-11-26T20:41:22,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742044_1220 (size=31255) 2024-11-26T20:41:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T20:41:22,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653742253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:22,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:22,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:22,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:22,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653742254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:22,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:22,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:22,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:22,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653742254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:22,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:22,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:22,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:22,490 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/1c593224e3b14492885899bb10d09567 2024-11-26T20:41:22,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/6848ef889a104e8ab6e31d4d98ad4879 is 50, key is test_row_0/B:col10/1732653679758/Put/seqid=0 2024-11-26T20:41:22,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742045_1221 (size=12301) 2024-11-26T20:41:22,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653742564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:22,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653742564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:22,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653742564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:22,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:22,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:22,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:22,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T20:41:22,905 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/6848ef889a104e8ab6e31d4d98ad4879 2024-11-26T20:41:22,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/3092927c794643b8ad327acce7511413 is 50, key is test_row_0/C:col10/1732653679758/Put/seqid=0 2024-11-26T20:41:22,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742046_1222 (size=12301) 2024-11-26T20:41:23,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653743074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:23,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:23,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653743074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:23,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653743074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:23,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:23,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:23,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:23,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:23,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:23,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:23,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:23,318 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/3092927c794643b8ad327acce7511413 2024-11-26T20:41:23,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/1c593224e3b14492885899bb10d09567 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/1c593224e3b14492885899bb10d09567 2024-11-26T20:41:23,333 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/1c593224e3b14492885899bb10d09567, entries=150, sequenceid=277, filesize=30.5 K 2024-11-26T20:41:23,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/6848ef889a104e8ab6e31d4d98ad4879 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/6848ef889a104e8ab6e31d4d98ad4879 2024-11-26T20:41:23,339 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/6848ef889a104e8ab6e31d4d98ad4879, entries=150, sequenceid=277, filesize=12.0 K 2024-11-26T20:41:23,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/3092927c794643b8ad327acce7511413 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/3092927c794643b8ad327acce7511413 2024-11-26T20:41:23,344 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/3092927c794643b8ad327acce7511413, entries=150, sequenceid=277, filesize=12.0 K 2024-11-26T20:41:23,346 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 00a77bb01b2de4ac52368f2c3e602b3a in 1687ms, sequenceid=277, compaction requested=false 2024-11-26T20:41:23,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:23,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:23,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-26T20:41:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-26T20:41:23,350 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-26T20:41:23,350 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8400 sec 2024-11-26T20:41:23,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.8470 sec 2024-11-26T20:41:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T20:41:23,654 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:23,655 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:41:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-26T20:41:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T20:41:23,657 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:41:23,658 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:41:23,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:41:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T20:41:23,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T20:41:23,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:23,811 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 00a77bb01b2de4ac52368f2c3e602b3a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T20:41:23,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=A 2024-11-26T20:41:23,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:23,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=B 2024-11-26T20:41:23,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:23,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=C 2024-11-26T20:41:23,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:23,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f68b145dd0854b8c9f66c7136d111bf2_00a77bb01b2de4ac52368f2c3e602b3a is 50, key is test_row_0/A:col10/1732653681933/Put/seqid=0 2024-11-26T20:41:23,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742047_1223 (size=12454) 2024-11-26T20:41:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T20:41:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:24,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:24,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653744114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:24,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653744115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:24,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653744120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:24,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:24,230 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f68b145dd0854b8c9f66c7136d111bf2_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f68b145dd0854b8c9f66c7136d111bf2_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:24,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/0d6fb566976b42b183334587be481636, store: [table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:24,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/0d6fb566976b42b183334587be481636 is 175, key is test_row_0/A:col10/1732653681933/Put/seqid=0 2024-11-26T20:41:24,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:24,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653744224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:24,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653744224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:24,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653744224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:24,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742048_1224 (size=31255) 2024-11-26T20:41:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T20:41:24,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653744434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:24,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:24,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:24,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653744444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:24,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653744444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:24,636 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/0d6fb566976b42b183334587be481636 2024-11-26T20:41:24,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/cee3f96d7be448ec9a01bbb5298ceb87 is 50, key is test_row_0/B:col10/1732653681933/Put/seqid=0 2024-11-26T20:41:24,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742049_1225 (size=12301) 2024-11-26T20:41:24,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653744744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:24,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653744753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:24,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:24,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653744753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:24,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:24,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T20:41:25,049 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/cee3f96d7be448ec9a01bbb5298ceb87 2024-11-26T20:41:25,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/dec8259f15fe47688bae0372f5b84082 is 50, key is test_row_0/C:col10/1732653681933/Put/seqid=0 2024-11-26T20:41:25,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742050_1226 (size=12301) 2024-11-26T20:41:25,062 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/dec8259f15fe47688bae0372f5b84082 2024-11-26T20:41:25,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/0d6fb566976b42b183334587be481636 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/0d6fb566976b42b183334587be481636 2024-11-26T20:41:25,071 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/0d6fb566976b42b183334587be481636, entries=150, sequenceid=294, filesize=30.5 K 2024-11-26T20:41:25,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/cee3f96d7be448ec9a01bbb5298ceb87 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/cee3f96d7be448ec9a01bbb5298ceb87 2024-11-26T20:41:25,082 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/cee3f96d7be448ec9a01bbb5298ceb87, entries=150, sequenceid=294, filesize=12.0 K 2024-11-26T20:41:25,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/dec8259f15fe47688bae0372f5b84082 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/dec8259f15fe47688bae0372f5b84082 2024-11-26T20:41:25,087 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/dec8259f15fe47688bae0372f5b84082, entries=150, sequenceid=294, filesize=12.0 K 2024-11-26T20:41:25,088 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 00a77bb01b2de4ac52368f2c3e602b3a in 1277ms, sequenceid=294, compaction requested=true 2024-11-26T20:41:25,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:25,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:25,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-26T20:41:25,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-26T20:41:25,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-26T20:41:25,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4340 sec 2024-11-26T20:41:25,096 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.4400 sec 2024-11-26T20:41:25,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:25,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00a77bb01b2de4ac52368f2c3e602b3a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T20:41:25,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=A 2024-11-26T20:41:25,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:25,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=B 2024-11-26T20:41:25,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:25,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=C 2024-11-26T20:41:25,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:25,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112694a73b7de5154f5e997d326a6c721218_00a77bb01b2de4ac52368f2c3e602b3a is 50, key is test_row_0/A:col10/1732653684118/Put/seqid=0 2024-11-26T20:41:25,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742051_1227 (size=14994) 2024-11-26T20:41:25,271 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:25,275 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112694a73b7de5154f5e997d326a6c721218_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112694a73b7de5154f5e997d326a6c721218_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:25,276 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/9233f36b9722430391f5c24c0edc416b, store: [table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:25,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/9233f36b9722430391f5c24c0edc416b is 175, key is test_row_0/A:col10/1732653684118/Put/seqid=0 2024-11-26T20:41:25,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742052_1228 (size=39949) 2024-11-26T20:41:25,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:25,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653745276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:25,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:25,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653745284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:25,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:25,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653745285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:25,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:25,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:25,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:25,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653745394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:25,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:25,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:25,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653745404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:25,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653745404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:25,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653745604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:25,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:25,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653745624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:25,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:25,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:25,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653745624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:25,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:25,681 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/9233f36b9722430391f5c24c0edc416b 2024-11-26T20:41:25,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/f4273db6ff1d4956ba8f178fb8153d1d is 50, key is test_row_0/B:col10/1732653684118/Put/seqid=0 2024-11-26T20:41:25,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742053_1229 (size=12301) 2024-11-26T20:41:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T20:41:25,804 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:25,804 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:41:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-26T20:41:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T20:41:25,806 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:41:25,807 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:41:25,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:41:25,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T20:41:25,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653745913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:25,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:25,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653745934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:25,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:25,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653745934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:25,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:25,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:25,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:25,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T20:41:25,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:25,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:25,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:25,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:25,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:25,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:26,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/f4273db6ff1d4956ba8f178fb8153d1d 2024-11-26T20:41:26,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/5a592a1b97f44c46adbedafa4990b4b6 is 50, key is test_row_0/C:col10/1732653684118/Put/seqid=0 2024-11-26T20:41:26,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742054_1230 (size=12301) 2024-11-26T20:41:26,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T20:41:26,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:26,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:26,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:26,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:26,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T20:41:26,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T20:41:26,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:26,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:26,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:26,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:26,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:26,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T20:41:26,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:26,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:26,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:26,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:26,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:26,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653746423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T20:41:26,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653746444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:26,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:26,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:26,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:26,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653746444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:26,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:26,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:26,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:26,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/5a592a1b97f44c46adbedafa4990b4b6 2024-11-26T20:41:26,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/9233f36b9722430391f5c24c0edc416b as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/9233f36b9722430391f5c24c0edc416b 2024-11-26T20:41:26,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/9233f36b9722430391f5c24c0edc416b, entries=200, sequenceid=313, filesize=39.0 K 2024-11-26T20:41:26,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/f4273db6ff1d4956ba8f178fb8153d1d as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/f4273db6ff1d4956ba8f178fb8153d1d 2024-11-26T20:41:26,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/f4273db6ff1d4956ba8f178fb8153d1d, entries=150, sequenceid=313, filesize=12.0 K 2024-11-26T20:41:26,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/5a592a1b97f44c46adbedafa4990b4b6 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/5a592a1b97f44c46adbedafa4990b4b6 2024-11-26T20:41:26,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/5a592a1b97f44c46adbedafa4990b4b6, entries=150, sequenceid=313, filesize=12.0 K 2024-11-26T20:41:26,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 00a77bb01b2de4ac52368f2c3e602b3a in 1275ms, sequenceid=313, compaction requested=true 2024-11-26T20:41:26,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:26,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a77bb01b2de4ac52368f2c3e602b3a:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:41:26,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:26,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a77bb01b2de4ac52368f2c3e602b3a:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:41:26,530 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:41:26,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:26,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a77bb01b2de4ac52368f2c3e602b3a:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:41:26,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:26,530 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:41:26,532 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49700 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:41:26,532 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134210 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:41:26,532 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 00a77bb01b2de4ac52368f2c3e602b3a/B is initiating minor compaction (all files) 2024-11-26T20:41:26,532 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 00a77bb01b2de4ac52368f2c3e602b3a/A is initiating minor compaction (all files) 2024-11-26T20:41:26,532 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a77bb01b2de4ac52368f2c3e602b3a/B in TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:26,532 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a77bb01b2de4ac52368f2c3e602b3a/A in TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:26,532 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/385f69cfa07b47f4b533cdc2c48b257b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/1c593224e3b14492885899bb10d09567, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/0d6fb566976b42b183334587be481636, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/9233f36b9722430391f5c24c0edc416b] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp, totalSize=131.1 K 2024-11-26T20:41:26,532 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/1c983009ba7948c1a4a5645a186cbeac, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/6848ef889a104e8ab6e31d4d98ad4879, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/cee3f96d7be448ec9a01bbb5298ceb87, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/f4273db6ff1d4956ba8f178fb8153d1d] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp, totalSize=48.5 K 2024-11-26T20:41:26,532 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:26,532 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. files: [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/385f69cfa07b47f4b533cdc2c48b257b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/1c593224e3b14492885899bb10d09567, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/0d6fb566976b42b183334587be481636, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/9233f36b9722430391f5c24c0edc416b] 2024-11-26T20:41:26,533 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 385f69cfa07b47f4b533cdc2c48b257b, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732653679090 2024-11-26T20:41:26,533 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c983009ba7948c1a4a5645a186cbeac, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732653679090 2024-11-26T20:41:26,533 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c593224e3b14492885899bb10d09567, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732653679755 2024-11-26T20:41:26,533 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 6848ef889a104e8ab6e31d4d98ad4879, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732653679755 2024-11-26T20:41:26,534 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting cee3f96d7be448ec9a01bbb5298ceb87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732653681932 2024-11-26T20:41:26,534 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d6fb566976b42b183334587be481636, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732653681932 2024-11-26T20:41:26,534 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9233f36b9722430391f5c24c0edc416b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732653684089 2024-11-26T20:41:26,534 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting f4273db6ff1d4956ba8f178fb8153d1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732653684107 2024-11-26T20:41:26,544 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:26,545 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a77bb01b2de4ac52368f2c3e602b3a#B#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:26,546 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/8654f3fbfc124ff8a38d987b99cbc66e is 50, key is test_row_0/B:col10/1732653684118/Put/seqid=0 2024-11-26T20:41:26,546 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411264371fb2ba5af4635ae16b09411d19647_00a77bb01b2de4ac52368f2c3e602b3a store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:26,549 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411264371fb2ba5af4635ae16b09411d19647_00a77bb01b2de4ac52368f2c3e602b3a, store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:26,549 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264371fb2ba5af4635ae16b09411d19647_00a77bb01b2de4ac52368f2c3e602b3a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:26,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742055_1231 (size=12983) 2024-11-26T20:41:26,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742056_1232 (size=4469) 2024-11-26T20:41:26,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T20:41:26,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:26,575 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 00a77bb01b2de4ac52368f2c3e602b3a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T20:41:26,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=A 2024-11-26T20:41:26,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:26,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=B 2024-11-26T20:41:26,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:26,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=C 2024-11-26T20:41:26,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:26,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a7546d0353b543fa8686a00d5705557b_00a77bb01b2de4ac52368f2c3e602b3a is 50, key is test_row_0/A:col10/1732653685282/Put/seqid=0 2024-11-26T20:41:26,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742057_1233 (size=12454) 2024-11-26T20:41:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T20:41:26,962 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/8654f3fbfc124ff8a38d987b99cbc66e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/8654f3fbfc124ff8a38d987b99cbc66e 2024-11-26T20:41:26,968 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00a77bb01b2de4ac52368f2c3e602b3a/B of 00a77bb01b2de4ac52368f2c3e602b3a into 8654f3fbfc124ff8a38d987b99cbc66e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:26,969 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:26,969 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., storeName=00a77bb01b2de4ac52368f2c3e602b3a/B, priority=12, startTime=1732653686530; duration=0sec 2024-11-26T20:41:26,969 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:26,969 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a77bb01b2de4ac52368f2c3e602b3a:B 2024-11-26T20:41:26,969 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:41:26,970 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a77bb01b2de4ac52368f2c3e602b3a#A#compaction#196 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:26,971 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/0b4536eeed3a468d8967081738879043 is 175, key is test_row_0/A:col10/1732653684118/Put/seqid=0 2024-11-26T20:41:26,972 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49700 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:41:26,972 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 00a77bb01b2de4ac52368f2c3e602b3a/C is initiating minor compaction (all files) 2024-11-26T20:41:26,972 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00a77bb01b2de4ac52368f2c3e602b3a/C in TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:26,972 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/74965d5d48494d5a8f496b6c63dc669f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/3092927c794643b8ad327acce7511413, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/dec8259f15fe47688bae0372f5b84082, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/5a592a1b97f44c46adbedafa4990b4b6] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp, totalSize=48.5 K 2024-11-26T20:41:26,972 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 74965d5d48494d5a8f496b6c63dc669f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732653679090 2024-11-26T20:41:26,973 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 3092927c794643b8ad327acce7511413, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732653679755 2024-11-26T20:41:26,974 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting dec8259f15fe47688bae0372f5b84082, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732653681932 2024-11-26T20:41:26,974 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a592a1b97f44c46adbedafa4990b4b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732653684107 2024-11-26T20:41:26,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742058_1234 (size=31937) 2024-11-26T20:41:26,983 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/0b4536eeed3a468d8967081738879043 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/0b4536eeed3a468d8967081738879043 2024-11-26T20:41:26,985 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00a77bb01b2de4ac52368f2c3e602b3a#C#compaction#198 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:26,986 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/e6e337435f65457a86d8fa993c006b67 is 50, key is test_row_0/C:col10/1732653684118/Put/seqid=0 2024-11-26T20:41:26,989 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00a77bb01b2de4ac52368f2c3e602b3a/A of 00a77bb01b2de4ac52368f2c3e602b3a into 0b4536eeed3a468d8967081738879043(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:26,989 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:26,989 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., storeName=00a77bb01b2de4ac52368f2c3e602b3a/A, priority=12, startTime=1732653686530; duration=0sec 2024-11-26T20:41:26,989 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:26,989 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a77bb01b2de4ac52368f2c3e602b3a:A 2024-11-26T20:41:26,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:26,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742059_1235 (size=12983) 2024-11-26T20:41:27,001 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a7546d0353b543fa8686a00d5705557b_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a7546d0353b543fa8686a00d5705557b_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:27,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/3eee3ab11aca4d56a6dbdc856f95bd71, store: [table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:27,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/3eee3ab11aca4d56a6dbdc856f95bd71 is 175, key is test_row_0/A:col10/1732653685282/Put/seqid=0 2024-11-26T20:41:27,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742060_1236 (size=31255) 2024-11-26T20:41:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:27,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. as already flushing 2024-11-26T20:41:27,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55170 deadline: 1732653747387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:27,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55242 deadline: 1732653747392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:27,399 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/e6e337435f65457a86d8fa993c006b67 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/e6e337435f65457a86d8fa993c006b67 2024-11-26T20:41:27,404 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00a77bb01b2de4ac52368f2c3e602b3a/C of 00a77bb01b2de4ac52368f2c3e602b3a into e6e337435f65457a86d8fa993c006b67(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:27,404 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:27,404 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., storeName=00a77bb01b2de4ac52368f2c3e602b3a/C, priority=12, startTime=1732653686530; duration=0sec 2024-11-26T20:41:27,404 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:27,404 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a77bb01b2de4ac52368f2c3e602b3a:C 2024-11-26T20:41:27,406 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/3eee3ab11aca4d56a6dbdc856f95bd71 2024-11-26T20:41:27,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/98c899675e72460fa2b8b40bd8c22b30 is 50, key is test_row_0/B:col10/1732653685282/Put/seqid=0 2024-11-26T20:41:27,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742061_1237 (size=12301) 2024-11-26T20:41:27,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:27,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55258 deadline: 1732653747434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:27,436 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 00a77bb01b2de4ac52368f2c3e602b3a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:41:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:27,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55196 deadline: 1732653747454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:27,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55180 deadline: 1732653747454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:27,457 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_2' in 00a77bb01b2de4ac52368f2c3e602b3a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:41:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:27,457 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 00a77bb01b2de4ac52368f2c3e602b3a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:41:27,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:27,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:27,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:27,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55170 deadline: 1732653747503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:27,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:27,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:27,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:27,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55242 deadline: 1732653747504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:27,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55170 deadline: 1732653747714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:27,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55242 deadline: 1732653747714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:27,740 INFO [Thread-741 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-741. 2024-11-26T20:41:27,740 DEBUG [Thread-741 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:41:27,740 DEBUG [Thread-741 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:27,740 INFO [Thread-743 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-743. 2024-11-26T20:41:27,740 DEBUG [Thread-743 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:41:27,740 DEBUG [Thread-743 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:27,740 DEBUG [Thread-741 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:27,740 DEBUG [Thread-743 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:27,740 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:41:27,740 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:41:27,741 INFO [Thread-745 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-745. 2024-11-26T20:41:27,741 DEBUG [Thread-745 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:41:27,741 DEBUG [Thread-745 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:27,741 DEBUG [Thread-745 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:27,741 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:41:27,747 INFO [Thread-747 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-747. 2024-11-26T20:41:27,747 DEBUG [Thread-747 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:41:27,747 DEBUG [Thread-747 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:27,748 DEBUG [Thread-747 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:27,748 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:41:27,818 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/98c899675e72460fa2b8b40bd8c22b30 2024-11-26T20:41:27,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/1c8339e2d54a465e95a60367fe614c27 is 50, key is test_row_0/C:col10/1732653685282/Put/seqid=0 2024-11-26T20:41:27,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742062_1238 (size=12301) 2024-11-26T20:41:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T20:41:28,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:28,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55170 deadline: 1732653748023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55242 deadline: 1732653748024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:28,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:28,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:28,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:28,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:28,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:28,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:41:28,229 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/1c8339e2d54a465e95a60367fe614c27 2024-11-26T20:41:28,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/3eee3ab11aca4d56a6dbdc856f95bd71 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3eee3ab11aca4d56a6dbdc856f95bd71 2024-11-26T20:41:28,237 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3eee3ab11aca4d56a6dbdc856f95bd71, entries=150, sequenceid=326, filesize=30.5 K 2024-11-26T20:41:28,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/98c899675e72460fa2b8b40bd8c22b30 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/98c899675e72460fa2b8b40bd8c22b30 2024-11-26T20:41:28,241 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/98c899675e72460fa2b8b40bd8c22b30, entries=150, sequenceid=326, filesize=12.0 K 2024-11-26T20:41:28,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/1c8339e2d54a465e95a60367fe614c27 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/1c8339e2d54a465e95a60367fe614c27 2024-11-26T20:41:28,245 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/1c8339e2d54a465e95a60367fe614c27, entries=150, sequenceid=326, filesize=12.0 K 2024-11-26T20:41:28,246 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 00a77bb01b2de4ac52368f2c3e602b3a in 1671ms, sequenceid=326, compaction requested=false 2024-11-26T20:41:28,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:28,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:28,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-26T20:41:28,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-26T20:41:28,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-26T20:41:28,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4400 sec 2024-11-26T20:41:28,250 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.4450 sec 2024-11-26T20:41:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:28,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00a77bb01b2de4ac52368f2c3e602b3a 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-26T20:41:28,536 INFO [Thread-738 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-738. 2024-11-26T20:41:28,536 INFO [Thread-734 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-734. 2024-11-26T20:41:28,536 DEBUG [Thread-734 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:41:28,536 DEBUG [Thread-738 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:41:28,536 DEBUG [Thread-734 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:28,536 DEBUG [Thread-738 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:28,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=A 2024-11-26T20:41:28,536 DEBUG [Thread-734 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:28,536 DEBUG [Thread-738 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:28,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:28,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=B 2024-11-26T20:41:28,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:28,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=C 2024-11-26T20:41:28,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:28,537 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:41:28,537 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:41:28,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112605b6eef4a1494c969450afe4cf631a06_00a77bb01b2de4ac52368f2c3e602b3a is 50, key is test_row_0/A:col10/1732653688534/Put/seqid=0 2024-11-26T20:41:28,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742063_1239 (size=12454) 2024-11-26T20:41:28,947 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:28,950 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112605b6eef4a1494c969450afe4cf631a06_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112605b6eef4a1494c969450afe4cf631a06_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:28,951 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/f9de907089124ac1815d142b1d884d6e, store: [table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:28,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/f9de907089124ac1815d142b1d884d6e is 175, key is test_row_0/A:col10/1732653688534/Put/seqid=0 2024-11-26T20:41:28,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742064_1240 (size=31255) 2024-11-26T20:41:29,355 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=350, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/f9de907089124ac1815d142b1d884d6e 2024-11-26T20:41:29,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/e6941d6e17a44355a43aab9778c17ef5 is 50, key is test_row_0/B:col10/1732653688534/Put/seqid=0 2024-11-26T20:41:29,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742065_1241 (size=12301) 2024-11-26T20:41:29,455 INFO [Thread-732 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-732. 2024-11-26T20:41:29,455 DEBUG [Thread-732 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:41:29,455 DEBUG [Thread-732 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:29,456 DEBUG [Thread-732 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:29,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:41:29,475 INFO [Thread-730 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-730. 2024-11-26T20:41:29,475 DEBUG [Thread-730 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:41:29,475 DEBUG [Thread-730 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:29,475 INFO [Thread-736 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-736. 2024-11-26T20:41:29,475 DEBUG [Thread-736 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:41:29,475 DEBUG [Thread-736 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:29,475 DEBUG [Thread-730 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:29,475 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:41:29,475 DEBUG [Thread-736 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:29,475 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:41:29,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/e6941d6e17a44355a43aab9778c17ef5 2024-11-26T20:41:29,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/6fa1e21a27504390a9aaf6fd5cc2cde7 is 50, key is test_row_0/C:col10/1732653688534/Put/seqid=0 2024-11-26T20:41:29,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742066_1242 (size=12301) 2024-11-26T20:41:29,966 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T20:41:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T20:41:29,974 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:29,974 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T20:41:29,974 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-26T20:41:29,974 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-26T20:41:29,974 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-26T20:41:29,974 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-26T20:41:29,974 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-26T20:41:29,974 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T20:41:29,974 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2577 2024-11-26T20:41:29,975 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2599 2024-11-26T20:41:29,975 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T20:41:29,975 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1149 2024-11-26T20:41:29,975 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3447 rows 2024-11-26T20:41:29,975 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1151 2024-11-26T20:41:29,975 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3453 rows 2024-11-26T20:41:29,975 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T20:41:29,975 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:41:29,975 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:29,975 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:29,975 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:41:29,976 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T20:41:29,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T20:41:29,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T20:41:29,979 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653689979"}]},"ts":"1732653689979"} 2024-11-26T20:41:29,981 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T20:41:29,981 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T20:41:29,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T20:41:29,983 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a77bb01b2de4ac52368f2c3e602b3a, UNASSIGN}] 2024-11-26T20:41:29,983 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a77bb01b2de4ac52368f2c3e602b3a, UNASSIGN 2024-11-26T20:41:29,984 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=00a77bb01b2de4ac52368f2c3e602b3a, regionState=CLOSING, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:41:29,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a77bb01b2de4ac52368f2c3e602b3a, UNASSIGN because future has completed 2024-11-26T20:41:29,985 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T20:41:29,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795}] 2024-11-26T20:41:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T20:41:30,138 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:30,138 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T20:41:30,138 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing 00a77bb01b2de4ac52368f2c3e602b3a, disabling compactions & flushes 2024-11-26T20:41:30,138 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:30,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/6fa1e21a27504390a9aaf6fd5cc2cde7 2024-11-26T20:41:30,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/f9de907089124ac1815d142b1d884d6e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/f9de907089124ac1815d142b1d884d6e 2024-11-26T20:41:30,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/f9de907089124ac1815d142b1d884d6e, entries=150, sequenceid=350, filesize=30.5 K 2024-11-26T20:41:30,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/e6941d6e17a44355a43aab9778c17ef5 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/e6941d6e17a44355a43aab9778c17ef5 2024-11-26T20:41:30,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/e6941d6e17a44355a43aab9778c17ef5, entries=150, sequenceid=350, filesize=12.0 K 2024-11-26T20:41:30,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/6fa1e21a27504390a9aaf6fd5cc2cde7 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/6fa1e21a27504390a9aaf6fd5cc2cde7 2024-11-26T20:41:30,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/6fa1e21a27504390a9aaf6fd5cc2cde7, entries=150, sequenceid=350, filesize=12.0 K 2024-11-26T20:41:30,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=20.13 KB/20610 for 00a77bb01b2de4ac52368f2c3e602b3a in 1660ms, sequenceid=350, compaction requested=true 2024-11-26T20:41:30,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00a77bb01b2de4ac52368f2c3e602b3a: 2024-11-26T20:41:30,196 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:30,196 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:30,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a77bb01b2de4ac52368f2c3e602b3a:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:41:30,196 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. after waiting 0 ms 2024-11-26T20:41:30,196 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:30,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:30,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a77bb01b2de4ac52368f2c3e602b3a:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:41:30,196 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing 00a77bb01b2de4ac52368f2c3e602b3a 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-26T20:41:30,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:30,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00a77bb01b2de4ac52368f2c3e602b3a:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:41:30,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:30,196 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. because compaction request was cancelled 2024-11-26T20:41:30,196 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a77bb01b2de4ac52368f2c3e602b3a:A 2024-11-26T20:41:30,197 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. because compaction request was cancelled 2024-11-26T20:41:30,197 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. because compaction request was cancelled 2024-11-26T20:41:30,197 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a77bb01b2de4ac52368f2c3e602b3a:C 2024-11-26T20:41:30,197 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00a77bb01b2de4ac52368f2c3e602b3a:B 2024-11-26T20:41:30,197 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=A 2024-11-26T20:41:30,197 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:30,197 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=B 2024-11-26T20:41:30,197 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:30,197 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00a77bb01b2de4ac52368f2c3e602b3a, store=C 2024-11-26T20:41:30,197 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:30,202 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ea9e144ec87f47bb9b71e497391062fc_00a77bb01b2de4ac52368f2c3e602b3a is 50, key is test_row_0/A:col10/1732653689474/Put/seqid=0 2024-11-26T20:41:30,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742067_1243 (size=9914) 2024-11-26T20:41:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T20:41:30,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T20:41:30,606 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:30,610 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ea9e144ec87f47bb9b71e497391062fc_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ea9e144ec87f47bb9b71e497391062fc_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:30,610 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/4e5a53a836a24f01ba991af3cb0529a7, store: [table=TestAcidGuarantees family=A region=00a77bb01b2de4ac52368f2c3e602b3a] 2024-11-26T20:41:30,611 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/4e5a53a836a24f01ba991af3cb0529a7 is 175, key is test_row_0/A:col10/1732653689474/Put/seqid=0 2024-11-26T20:41:30,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742068_1244 (size=22561) 2024-11-26T20:41:31,016 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/4e5a53a836a24f01ba991af3cb0529a7 2024-11-26T20:41:31,024 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/87d6c97ca3994595b091c776d9cead2a is 50, key is test_row_0/B:col10/1732653689474/Put/seqid=0 2024-11-26T20:41:31,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742069_1245 (size=9857) 2024-11-26T20:41:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T20:41:31,429 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/87d6c97ca3994595b091c776d9cead2a 2024-11-26T20:41:31,438 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/dab59d71b6494e0fa41f2fda11a90329 is 50, key is test_row_0/C:col10/1732653689474/Put/seqid=0 2024-11-26T20:41:31,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742070_1246 (size=9857) 2024-11-26T20:41:31,843 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/dab59d71b6494e0fa41f2fda11a90329 2024-11-26T20:41:31,850 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/A/4e5a53a836a24f01ba991af3cb0529a7 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/4e5a53a836a24f01ba991af3cb0529a7 2024-11-26T20:41:31,855 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/4e5a53a836a24f01ba991af3cb0529a7, entries=100, sequenceid=356, filesize=22.0 K 2024-11-26T20:41:31,856 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/B/87d6c97ca3994595b091c776d9cead2a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/87d6c97ca3994595b091c776d9cead2a 2024-11-26T20:41:31,860 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/87d6c97ca3994595b091c776d9cead2a, entries=100, sequenceid=356, filesize=9.6 K 2024-11-26T20:41:31,861 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/.tmp/C/dab59d71b6494e0fa41f2fda11a90329 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/dab59d71b6494e0fa41f2fda11a90329 2024-11-26T20:41:31,866 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/dab59d71b6494e0fa41f2fda11a90329, entries=100, sequenceid=356, filesize=9.6 K 2024-11-26T20:41:31,867 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 00a77bb01b2de4ac52368f2c3e602b3a in 1671ms, sequenceid=356, compaction requested=true 2024-11-26T20:41:31,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3fd06b044e0d447a8f5b9ad7ebf667cc, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/a97b1c10e3ca4b5dbea59a05a71732a8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3591f216b9144464bdd9d738f3efa41f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/824fd357f5594d99a3eaf72f2033bbb9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/1937d1de1b814aaeb842bf718d45927c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/6c642fd05ec3485e9dee65a14a43b930, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3efa590d90c64296813b4074e43f74e7, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/9b86f9c2e6d2490ea9b75a59314f0e23, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/ec896b442e4a4697b0b0490ef44dda35, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/807ef4a93cdc4f6aa38f9fd9b0889e4b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/48228e3ed9bf40f7bc925b54117a2f11, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/655505aabdd94d41b3f47ac99f902cd1, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/a3710bb1d26f4e9485367fbf624a9936, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/df82671c56d540f2a59a9f9615220be2, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/31fa73e13cb442318a59279a70b49c4f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/9a742c1f5b5a474f92826cfbe4286d31, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/385f69cfa07b47f4b533cdc2c48b257b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/1c593224e3b14492885899bb10d09567, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/0d6fb566976b42b183334587be481636, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/9233f36b9722430391f5c24c0edc416b] to archive 2024-11-26T20:41:31,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T20:41:31,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3fd06b044e0d447a8f5b9ad7ebf667cc to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3fd06b044e0d447a8f5b9ad7ebf667cc 2024-11-26T20:41:31,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/a97b1c10e3ca4b5dbea59a05a71732a8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/a97b1c10e3ca4b5dbea59a05a71732a8 2024-11-26T20:41:31,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3591f216b9144464bdd9d738f3efa41f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3591f216b9144464bdd9d738f3efa41f 2024-11-26T20:41:31,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/824fd357f5594d99a3eaf72f2033bbb9 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/824fd357f5594d99a3eaf72f2033bbb9 2024-11-26T20:41:31,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/1937d1de1b814aaeb842bf718d45927c to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/1937d1de1b814aaeb842bf718d45927c 2024-11-26T20:41:31,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/6c642fd05ec3485e9dee65a14a43b930 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/6c642fd05ec3485e9dee65a14a43b930 2024-11-26T20:41:31,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3efa590d90c64296813b4074e43f74e7 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3efa590d90c64296813b4074e43f74e7 2024-11-26T20:41:31,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/9b86f9c2e6d2490ea9b75a59314f0e23 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/9b86f9c2e6d2490ea9b75a59314f0e23 2024-11-26T20:41:31,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/ec896b442e4a4697b0b0490ef44dda35 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/ec896b442e4a4697b0b0490ef44dda35 2024-11-26T20:41:31,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/807ef4a93cdc4f6aa38f9fd9b0889e4b to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/807ef4a93cdc4f6aa38f9fd9b0889e4b 2024-11-26T20:41:31,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/48228e3ed9bf40f7bc925b54117a2f11 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/48228e3ed9bf40f7bc925b54117a2f11 2024-11-26T20:41:31,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/655505aabdd94d41b3f47ac99f902cd1 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/655505aabdd94d41b3f47ac99f902cd1 2024-11-26T20:41:31,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/a3710bb1d26f4e9485367fbf624a9936 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/a3710bb1d26f4e9485367fbf624a9936 2024-11-26T20:41:31,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/df82671c56d540f2a59a9f9615220be2 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/df82671c56d540f2a59a9f9615220be2 2024-11-26T20:41:31,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/31fa73e13cb442318a59279a70b49c4f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/31fa73e13cb442318a59279a70b49c4f 2024-11-26T20:41:31,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/9a742c1f5b5a474f92826cfbe4286d31 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/9a742c1f5b5a474f92826cfbe4286d31 2024-11-26T20:41:31,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/385f69cfa07b47f4b533cdc2c48b257b to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/385f69cfa07b47f4b533cdc2c48b257b 2024-11-26T20:41:31,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/1c593224e3b14492885899bb10d09567 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/1c593224e3b14492885899bb10d09567 2024-11-26T20:41:31,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/0d6fb566976b42b183334587be481636 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/0d6fb566976b42b183334587be481636 2024-11-26T20:41:31,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/9233f36b9722430391f5c24c0edc416b to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/9233f36b9722430391f5c24c0edc416b 2024-11-26T20:41:31,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/9f835546920f41cbb30df4a8d2781032, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/3d25af26302e4bc2a0a9848a21d72c91, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/4f473b53d502433d9e04d8a005fe269c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/b62a41f4b859410f9e00c4425c735415, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/303fcfe6bbb547f184e8066e5363a5e1, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/3d72ee9775054dbe906fa6c7617c7af9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/a5cf713272c74b578501535eabdd4745, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/4c678b100a444246bdde0d4fafefde6d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/982eebfddbdb4028932326c5da241b78, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/85910462b30f41b3a1109b80fab70c6d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/2170500e763246e7a74fecb33e2802ef, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/83f3cf3f210041eda6e6f1910fb5fa39, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/fb3dc7075a614a71aea43cc836846b14, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/1e6ae85c9efd4fdca64d97d1221b6aae, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/8f36750e3eb6403382cd77efdd4a5d8c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/1c983009ba7948c1a4a5645a186cbeac, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/59d5d6a0b90c4819a2510539bf7ea69a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/6848ef889a104e8ab6e31d4d98ad4879, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/cee3f96d7be448ec9a01bbb5298ceb87, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/f4273db6ff1d4956ba8f178fb8153d1d] to archive 2024-11-26T20:41:31,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T20:41:31,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/9f835546920f41cbb30df4a8d2781032 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/9f835546920f41cbb30df4a8d2781032 2024-11-26T20:41:31,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/3d25af26302e4bc2a0a9848a21d72c91 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/3d25af26302e4bc2a0a9848a21d72c91 2024-11-26T20:41:31,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/4f473b53d502433d9e04d8a005fe269c to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/4f473b53d502433d9e04d8a005fe269c 2024-11-26T20:41:31,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/b62a41f4b859410f9e00c4425c735415 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/b62a41f4b859410f9e00c4425c735415 2024-11-26T20:41:31,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/303fcfe6bbb547f184e8066e5363a5e1 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/303fcfe6bbb547f184e8066e5363a5e1 2024-11-26T20:41:31,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/3d72ee9775054dbe906fa6c7617c7af9 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/3d72ee9775054dbe906fa6c7617c7af9 2024-11-26T20:41:31,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/a5cf713272c74b578501535eabdd4745 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/a5cf713272c74b578501535eabdd4745 2024-11-26T20:41:31,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/4c678b100a444246bdde0d4fafefde6d to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/4c678b100a444246bdde0d4fafefde6d 2024-11-26T20:41:31,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/982eebfddbdb4028932326c5da241b78 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/982eebfddbdb4028932326c5da241b78 2024-11-26T20:41:31,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/85910462b30f41b3a1109b80fab70c6d to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/85910462b30f41b3a1109b80fab70c6d 2024-11-26T20:41:31,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/2170500e763246e7a74fecb33e2802ef to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/2170500e763246e7a74fecb33e2802ef 2024-11-26T20:41:31,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/83f3cf3f210041eda6e6f1910fb5fa39 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/83f3cf3f210041eda6e6f1910fb5fa39 2024-11-26T20:41:31,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/fb3dc7075a614a71aea43cc836846b14 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/fb3dc7075a614a71aea43cc836846b14 2024-11-26T20:41:31,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/1e6ae85c9efd4fdca64d97d1221b6aae to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/1e6ae85c9efd4fdca64d97d1221b6aae 2024-11-26T20:41:31,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/8f36750e3eb6403382cd77efdd4a5d8c to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/8f36750e3eb6403382cd77efdd4a5d8c 2024-11-26T20:41:31,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/1c983009ba7948c1a4a5645a186cbeac to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/1c983009ba7948c1a4a5645a186cbeac 2024-11-26T20:41:31,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/59d5d6a0b90c4819a2510539bf7ea69a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/59d5d6a0b90c4819a2510539bf7ea69a 2024-11-26T20:41:31,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/6848ef889a104e8ab6e31d4d98ad4879 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/6848ef889a104e8ab6e31d4d98ad4879 2024-11-26T20:41:31,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/cee3f96d7be448ec9a01bbb5298ceb87 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/cee3f96d7be448ec9a01bbb5298ceb87 2024-11-26T20:41:31,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/f4273db6ff1d4956ba8f178fb8153d1d to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/f4273db6ff1d4956ba8f178fb8153d1d 2024-11-26T20:41:31,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/29448f06910c4989961d0edca282cb79, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/af7b01b219d24494a5d16dbb6c4d8a34, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/656257f9031645ab81e507f1835c2146, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/4a58a81a2fee490a94d882ecb768f6a4, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/817f9dd8f89f4707a228fae7c1d22879, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/ed4329e2bdc044cf99d0c5caa8d60cf1, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/3a46978eb63c4284a1e0e0a60f108b98, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/047e2a23b7224ffd92f737ff2c4982dd, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/d0efaf9c27494c999e735faa40c07ab9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/d6f53556cff64bbe81c72f9d6a29f926, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/4f9c81cc6eec463e9f7164e4ffadfd62, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/f3a8ce8f70464258be5c985315292d45, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/57ead70af2934c39aac9dc9ee0579ea9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/b170545f5c77496bbca658d6d9cc7fb6, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/73406d71a1834e80adeb6a43d9752c06, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/74965d5d48494d5a8f496b6c63dc669f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/4767b7e18b994c57aa42d452b2e3a79b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/3092927c794643b8ad327acce7511413, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/dec8259f15fe47688bae0372f5b84082, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/5a592a1b97f44c46adbedafa4990b4b6] to archive 2024-11-26T20:41:31,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T20:41:31,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/29448f06910c4989961d0edca282cb79 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/29448f06910c4989961d0edca282cb79 2024-11-26T20:41:31,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/af7b01b219d24494a5d16dbb6c4d8a34 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/af7b01b219d24494a5d16dbb6c4d8a34 2024-11-26T20:41:31,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/656257f9031645ab81e507f1835c2146 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/656257f9031645ab81e507f1835c2146 2024-11-26T20:41:31,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/4a58a81a2fee490a94d882ecb768f6a4 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/4a58a81a2fee490a94d882ecb768f6a4 2024-11-26T20:41:31,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/817f9dd8f89f4707a228fae7c1d22879 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/817f9dd8f89f4707a228fae7c1d22879 2024-11-26T20:41:31,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/ed4329e2bdc044cf99d0c5caa8d60cf1 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/ed4329e2bdc044cf99d0c5caa8d60cf1 2024-11-26T20:41:31,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/3a46978eb63c4284a1e0e0a60f108b98 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/3a46978eb63c4284a1e0e0a60f108b98 2024-11-26T20:41:31,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/047e2a23b7224ffd92f737ff2c4982dd to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/047e2a23b7224ffd92f737ff2c4982dd 2024-11-26T20:41:31,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/d0efaf9c27494c999e735faa40c07ab9 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/d0efaf9c27494c999e735faa40c07ab9 2024-11-26T20:41:31,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/d6f53556cff64bbe81c72f9d6a29f926 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/d6f53556cff64bbe81c72f9d6a29f926 2024-11-26T20:41:31,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/4f9c81cc6eec463e9f7164e4ffadfd62 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/4f9c81cc6eec463e9f7164e4ffadfd62 2024-11-26T20:41:31,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/f3a8ce8f70464258be5c985315292d45 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/f3a8ce8f70464258be5c985315292d45 2024-11-26T20:41:31,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/57ead70af2934c39aac9dc9ee0579ea9 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/57ead70af2934c39aac9dc9ee0579ea9 2024-11-26T20:41:31,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/b170545f5c77496bbca658d6d9cc7fb6 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/b170545f5c77496bbca658d6d9cc7fb6 2024-11-26T20:41:31,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/73406d71a1834e80adeb6a43d9752c06 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/73406d71a1834e80adeb6a43d9752c06 2024-11-26T20:41:31,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/74965d5d48494d5a8f496b6c63dc669f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/74965d5d48494d5a8f496b6c63dc669f 2024-11-26T20:41:31,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/4767b7e18b994c57aa42d452b2e3a79b to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/4767b7e18b994c57aa42d452b2e3a79b 2024-11-26T20:41:31,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/3092927c794643b8ad327acce7511413 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/3092927c794643b8ad327acce7511413 2024-11-26T20:41:31,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/dec8259f15fe47688bae0372f5b84082 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/dec8259f15fe47688bae0372f5b84082 2024-11-26T20:41:31,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/5a592a1b97f44c46adbedafa4990b4b6 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/5a592a1b97f44c46adbedafa4990b4b6 2024-11-26T20:41:31,964 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/recovered.edits/359.seqid, newMaxSeqId=359, maxSeqId=4 2024-11-26T20:41:31,965 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. 2024-11-26T20:41:31,965 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for 00a77bb01b2de4ac52368f2c3e602b3a: Waiting for close lock at 1732653690138Running coprocessor pre-close hooks at 1732653690138Disabling compacts and flushes for region at 1732653690138Disabling writes for close at 1732653690196 (+58 ms)Obtaining lock to block concurrent updates at 1732653690196Preparing flush snapshotting stores in 00a77bb01b2de4ac52368f2c3e602b3a at 1732653690196Finished memstore snapshotting TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732653690197 (+1 ms)Flushing stores of TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a. at 1732653690197Flushing 00a77bb01b2de4ac52368f2c3e602b3a/A: creating writer at 1732653690197Flushing mob file 00a77bb01b2de4ac52368f2c3e602b3a/A: appending metadata at 1732653690202 (+5 ms)Flushing mob file 00a77bb01b2de4ac52368f2c3e602b3a/A: closing flushed file at 1732653690202Flushing 00a77bb01b2de4ac52368f2c3e602b3a/A: appending metadata at 1732653690611 (+409 ms)Flushing 00a77bb01b2de4ac52368f2c3e602b3a/A: closing flushed file at 1732653690611Flushing 00a77bb01b2de4ac52368f2c3e602b3a/B: creating writer at 1732653691020 (+409 ms)Flushing 00a77bb01b2de4ac52368f2c3e602b3a/B: appending metadata at 1732653691023 (+3 ms)Flushing 00a77bb01b2de4ac52368f2c3e602b3a/B: closing flushed file at 1732653691024 (+1 ms)Flushing 00a77bb01b2de4ac52368f2c3e602b3a/C: creating writer at 1732653691434 (+410 ms)Flushing 00a77bb01b2de4ac52368f2c3e602b3a/C: appending metadata at 1732653691438 (+4 ms)Flushing 00a77bb01b2de4ac52368f2c3e602b3a/C: closing flushed file at 1732653691438Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2365264: reopening flushed file at 1732653691849 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f2a378d: reopening flushed file at 1732653691855 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c7b8d5a: reopening flushed file at 1732653691860 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 00a77bb01b2de4ac52368f2c3e602b3a in 1671ms, sequenceid=356, compaction requested=true at 1732653691867 (+7 ms)Writing region close event to WAL at 1732653691961 (+94 ms)Running coprocessor post-close hooks at 1732653691965 (+4 ms)Closed at 1732653691965 2024-11-26T20:41:31,967 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed 00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:31,968 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=00a77bb01b2de4ac52368f2c3e602b3a, regionState=CLOSED 2024-11-26T20:41:31,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 because future has completed 2024-11-26T20:41:31,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-26T20:41:31,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure 00a77bb01b2de4ac52368f2c3e602b3a, server=cd6be161543e,40291,1732653631795 in 1.9850 sec 2024-11-26T20:41:31,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-26T20:41:31,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00a77bb01b2de4ac52368f2c3e602b3a, UNASSIGN in 1.9890 sec 2024-11-26T20:41:31,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-26T20:41:31,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9930 sec 2024-11-26T20:41:31,977 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653691976"}]},"ts":"1732653691976"} 2024-11-26T20:41:31,978 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T20:41:31,978 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T20:41:31,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0030 sec 2024-11-26T20:41:32,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T20:41:32,134 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:32,135 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T20:41:32,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:41:32,137 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:41:32,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T20:41:32,138 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=66, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:41:32,142 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:32,144 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A, FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B, FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C, FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/recovered.edits] 2024-11-26T20:41:32,147 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/0b4536eeed3a468d8967081738879043 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/0b4536eeed3a468d8967081738879043 2024-11-26T20:41:32,149 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3eee3ab11aca4d56a6dbdc856f95bd71 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/3eee3ab11aca4d56a6dbdc856f95bd71 2024-11-26T20:41:32,150 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/4e5a53a836a24f01ba991af3cb0529a7 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/4e5a53a836a24f01ba991af3cb0529a7 2024-11-26T20:41:32,151 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/f9de907089124ac1815d142b1d884d6e to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/A/f9de907089124ac1815d142b1d884d6e 2024-11-26T20:41:32,154 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/8654f3fbfc124ff8a38d987b99cbc66e to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/8654f3fbfc124ff8a38d987b99cbc66e 2024-11-26T20:41:32,155 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/87d6c97ca3994595b091c776d9cead2a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/87d6c97ca3994595b091c776d9cead2a 2024-11-26T20:41:32,157 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/98c899675e72460fa2b8b40bd8c22b30 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/98c899675e72460fa2b8b40bd8c22b30 2024-11-26T20:41:32,158 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/e6941d6e17a44355a43aab9778c17ef5 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/B/e6941d6e17a44355a43aab9778c17ef5 2024-11-26T20:41:32,161 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/1c8339e2d54a465e95a60367fe614c27 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/1c8339e2d54a465e95a60367fe614c27 2024-11-26T20:41:32,162 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/6fa1e21a27504390a9aaf6fd5cc2cde7 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/6fa1e21a27504390a9aaf6fd5cc2cde7 2024-11-26T20:41:32,163 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/dab59d71b6494e0fa41f2fda11a90329 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/dab59d71b6494e0fa41f2fda11a90329 2024-11-26T20:41:32,165 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/e6e337435f65457a86d8fa993c006b67 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/C/e6e337435f65457a86d8fa993c006b67 2024-11-26T20:41:32,168 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/recovered.edits/359.seqid to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a/recovered.edits/359.seqid 2024-11-26T20:41:32,169 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:32,169 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T20:41:32,169 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T20:41:32,170 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-26T20:41:32,174 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112605b6eef4a1494c969450afe4cf631a06_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112605b6eef4a1494c969450afe4cf631a06_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:32,175 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126188182a9d0f64ce3bfac5beba8c87483_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126188182a9d0f64ce3bfac5beba8c87483_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:32,177 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126340ab04c1b1c49a4af8e5691439ea8c5_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126340ab04c1b1c49a4af8e5691439ea8c5_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:32,178 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263a6a61e5b10a438ea8a13fcea5f4fae4_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263a6a61e5b10a438ea8a13fcea5f4fae4_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:32,179 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112650864314e1ad466a9b3b9276e19a28a1_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112650864314e1ad466a9b3b9276e19a28a1_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:32,180 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126844a2ca7fd0d412092b591c64d2976d7_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126844a2ca7fd0d412092b591c64d2976d7_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:32,182 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112684639e58491941d4828f353187fc3be8_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112684639e58491941d4828f353187fc3be8_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:32,183 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112694a73b7de5154f5e997d326a6c721218_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112694a73b7de5154f5e997d326a6c721218_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:32,184 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a28f2881646d43a888d7dad0186fbfb1_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a28f2881646d43a888d7dad0186fbfb1_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:32,185 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a7546d0353b543fa8686a00d5705557b_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a7546d0353b543fa8686a00d5705557b_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:32,187 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126abd5536b141547219a33fdc201b35d26_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126abd5536b141547219a33fdc201b35d26_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:32,188 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c7faa3ffac894464b2986898a4d1112b_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c7faa3ffac894464b2986898a4d1112b_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:32,189 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ce91974721854229878054aa5fa76323_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ce91974721854229878054aa5fa76323_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:32,190 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ea9e144ec87f47bb9b71e497391062fc_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ea9e144ec87f47bb9b71e497391062fc_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:32,192 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f1212472e3d5470385e153a834db8d0f_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f1212472e3d5470385e153a834db8d0f_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:32,193 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f68b145dd0854b8c9f66c7136d111bf2_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f68b145dd0854b8c9f66c7136d111bf2_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:32,194 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f78cf6959eb74d079ac287f2465bf8fe_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f78cf6959eb74d079ac287f2465bf8fe_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:32,195 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126fe76645d76ed4911aa7b1fa900f65aac_00a77bb01b2de4ac52368f2c3e602b3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126fe76645d76ed4911aa7b1fa900f65aac_00a77bb01b2de4ac52368f2c3e602b3a 2024-11-26T20:41:32,196 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T20:41:32,198 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=66, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:41:32,201 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T20:41:32,203 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T20:41:32,204 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=66, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:41:32,204 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T20:41:32,205 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732653692204"}]},"ts":"9223372036854775807"} 2024-11-26T20:41:32,209 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T20:41:32,209 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 00a77bb01b2de4ac52368f2c3e602b3a, NAME => 'TestAcidGuarantees,,1732653665905.00a77bb01b2de4ac52368f2c3e602b3a.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T20:41:32,209 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T20:41:32,209 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732653692209"}]},"ts":"9223372036854775807"} 2024-11-26T20:41:32,212 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T20:41:32,212 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=66, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:41:32,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 77 msec 2024-11-26T20:41:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T20:41:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T20:41:32,245 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:32,255 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 247) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1445524182_22 at /127.0.0.1:50856 [Waiting for operation #489] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/cluster_bc121de7-746a-3d7e-5c7c-57aa2e103a17/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/cluster_bc121de7-746a-3d7e-5c7c-57aa2e103a17/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1770967530_22 at /127.0.0.1:52304 [Waiting for operation #616] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1770967530_22 at /127.0.0.1:52370 [Waiting for operation #622] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1445524182_22 at /127.0.0.1:52474 [Waiting for operation #575] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=514 (was 439) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6417 (was 5848) - AvailableMemoryMB LEAK? - 2024-11-26T20:41:32,266 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=514, ProcessCount=11, AvailableMemoryMB=6401 2024-11-26T20:41:32,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T20:41:32,267 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T20:41:32,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T20:41:32,269 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T20:41:32,269 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:32,269 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 67 2024-11-26T20:41:32,270 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T20:41:32,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-26T20:41:32,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742071_1247 (size=1061) 2024-11-26T20:41:32,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-26T20:41:32,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-26T20:41:32,681 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794 2024-11-26T20:41:32,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742072_1248 (size=53) 2024-11-26T20:41:32,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-26T20:41:33,088 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:41:33,088 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 38de20de4b25bae04950a5d15dc346de, disabling compactions & flushes 2024-11-26T20:41:33,088 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:33,088 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:33,088 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. after waiting 0 ms 2024-11-26T20:41:33,088 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:33,088 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:33,088 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 38de20de4b25bae04950a5d15dc346de: Waiting for close lock at 1732653693088Disabling compacts and flushes for region at 1732653693088Disabling writes for close at 1732653693088Writing region close event to WAL at 1732653693088Closed at 1732653693088 2024-11-26T20:41:33,089 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T20:41:33,089 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732653693089"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732653693089"}]},"ts":"1732653693089"} 2024-11-26T20:41:33,091 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T20:41:33,091 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T20:41:33,091 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653693091"}]},"ts":"1732653693091"} 2024-11-26T20:41:33,092 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T20:41:33,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38de20de4b25bae04950a5d15dc346de, ASSIGN}] 2024-11-26T20:41:33,094 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38de20de4b25bae04950a5d15dc346de, ASSIGN 2024-11-26T20:41:33,095 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=38de20de4b25bae04950a5d15dc346de, ASSIGN; state=OFFLINE, location=cd6be161543e,40291,1732653631795; forceNewPlan=false, retain=false 2024-11-26T20:41:33,246 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=38de20de4b25bae04950a5d15dc346de, regionState=OPENING, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:41:33,247 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=38de20de4b25bae04950a5d15dc346de, ASSIGN because future has completed 2024-11-26T20:41:33,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795}] 2024-11-26T20:41:33,403 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:33,403 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7752): Opening region: {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:41:33,404 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:33,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-26T20:41:33,404 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:41:33,404 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7794): checking encryption for 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:33,404 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7797): checking classloading for 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:33,405 INFO [StoreOpener-38de20de4b25bae04950a5d15dc346de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:33,406 INFO [StoreOpener-38de20de4b25bae04950a5d15dc346de-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:41:33,406 INFO [StoreOpener-38de20de4b25bae04950a5d15dc346de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38de20de4b25bae04950a5d15dc346de columnFamilyName A 2024-11-26T20:41:33,407 DEBUG [StoreOpener-38de20de4b25bae04950a5d15dc346de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:33,407 INFO [StoreOpener-38de20de4b25bae04950a5d15dc346de-1 {}] regionserver.HStore(327): Store=38de20de4b25bae04950a5d15dc346de/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:41:33,407 INFO [StoreOpener-38de20de4b25bae04950a5d15dc346de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:33,408 INFO [StoreOpener-38de20de4b25bae04950a5d15dc346de-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:41:33,408 INFO [StoreOpener-38de20de4b25bae04950a5d15dc346de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38de20de4b25bae04950a5d15dc346de columnFamilyName B 2024-11-26T20:41:33,408 DEBUG [StoreOpener-38de20de4b25bae04950a5d15dc346de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:33,409 INFO [StoreOpener-38de20de4b25bae04950a5d15dc346de-1 {}] regionserver.HStore(327): Store=38de20de4b25bae04950a5d15dc346de/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:41:33,409 INFO [StoreOpener-38de20de4b25bae04950a5d15dc346de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:33,410 INFO [StoreOpener-38de20de4b25bae04950a5d15dc346de-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:41:33,410 INFO [StoreOpener-38de20de4b25bae04950a5d15dc346de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38de20de4b25bae04950a5d15dc346de columnFamilyName C 2024-11-26T20:41:33,410 DEBUG [StoreOpener-38de20de4b25bae04950a5d15dc346de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:41:33,411 INFO [StoreOpener-38de20de4b25bae04950a5d15dc346de-1 {}] regionserver.HStore(327): Store=38de20de4b25bae04950a5d15dc346de/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:41:33,411 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:33,411 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1038): replaying wal for 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:33,412 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:33,412 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:33,413 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1048): stopping wal replay for 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:33,413 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1060): Cleaning up temporary data for 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:33,414 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T20:41:33,415 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1093): writing seq id for 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:33,417 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:41:33,418 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1114): Opened 38de20de4b25bae04950a5d15dc346de; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62800332, jitterRate=-0.06420212984085083}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T20:41:33,418 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:33,418 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1006): Region open journal for 38de20de4b25bae04950a5d15dc346de: Running coprocessor pre-open hook at 1732653693404Writing region info on filesystem at 1732653693404Initializing all the Stores at 1732653693405 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653693405Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653693405Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653693405Cleaning up temporary data from old regions at 1732653693413 (+8 ms)Running coprocessor post-open hooks at 1732653693418 (+5 ms)Region opened successfully at 1732653693418 2024-11-26T20:41:33,419 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., pid=69, masterSystemTime=1732653693399 2024-11-26T20:41:33,421 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:33,421 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:33,421 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=38de20de4b25bae04950a5d15dc346de, regionState=OPEN, openSeqNum=2, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:41:33,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 because future has completed 2024-11-26T20:41:33,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-26T20:41:33,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 in 175 msec 2024-11-26T20:41:33,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=67 2024-11-26T20:41:33,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=67, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38de20de4b25bae04950a5d15dc346de, ASSIGN in 332 msec 2024-11-26T20:41:33,427 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T20:41:33,427 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653693427"}]},"ts":"1732653693427"} 2024-11-26T20:41:33,428 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T20:41:33,429 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T20:41:33,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1620 sec 2024-11-26T20:41:34,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-26T20:41:34,415 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:34,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3260768f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:41:34,417 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:41:34,418 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:41:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:41:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:41:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1da4f640, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:41:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:41:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:34,420 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:41:34,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42f392af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,421 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:41:34,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:41:34,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:41:34,424 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:41:34,425 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T20:41:34,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is cd6be161543e,42359,1732653630959 2024-11-26T20:41:34,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@371461de 2024-11-26T20:41:34,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T20:41:34,427 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T20:41:34,429 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555b0fec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,429 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:41:34,429 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:41:34,430 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:41:34,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:41:34,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:41:34,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40436af5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:41:34,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:41:34,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:34,432 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:41:34,433 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10c3301, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,433 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ad36e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,433 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:41:34,434 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:41:34,434 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:41:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:41:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:41:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b2e17f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:41:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:41:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:34,436 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:41:34,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7907d48a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b38a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,438 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:41:34,438 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:41:34,438 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:41:34,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:41:34,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:41:34,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fb0b422, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:41:34,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:41:34,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:34,440 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:41:34,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ad0ffd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260eab4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,441 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:41:34,442 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:41:34,442 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:41:34,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:41:34,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:41:34,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56cacb85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:41:34,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:41:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:34,444 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58382, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:41:34,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60e20a31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699a7f66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,446 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:41:34,446 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:41:34,447 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:41:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:41:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:41:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c9b31fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:41:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:41:34,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:34,449 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:41:34,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@274eb770, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19903550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,450 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:41:34,451 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:41:34,451 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:41:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:41:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:41:34,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca331ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:41:34,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:41:34,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:34,453 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:41:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58b75744, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8815e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,454 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:41:34,455 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:41:34,455 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:41:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:41:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:41:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15cc5e43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:41:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:41:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:34,457 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:41:34,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43d48253, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ed310c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,460 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:41:34,461 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:41:34,461 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:41:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:41:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:41:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37dff720, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:41:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:41:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:34,463 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:41:34,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c9bd1fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176a9725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,464 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:41:34,464 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:41:34,465 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:41:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:41:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:41:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@157a9629, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:41:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:41:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:34,466 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:41:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a095cca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,467 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5762fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,467 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:41:34,468 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:41:34,468 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:41:34,468 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:41:34,468 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:41:34,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7732c9a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:41:34,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:41:34,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:34,470 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:41:34,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eb3e077, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:41:34,473 DEBUG [Thread-1164 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:41:34,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:41:34,473 DEBUG [Thread-1166 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:41:34,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:41:34,474 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:41:34,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:41:34,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:41:34,475 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:41:34,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:41:34,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:41:34,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-26T20:41:34,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T20:41:34,479 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:41:34,480 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:41:34,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:41:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:41:34,489 DEBUG [Thread-1151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:41:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:41:34,490 DEBUG [Thread-1151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:41:34,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:41:34,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:41:34,508 DEBUG [Thread-1153 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:41:34,509 DEBUG [Thread-1170 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:41:34,510 DEBUG [Thread-1155 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:41:34,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:41:34,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:41:34,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:41:34,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:41:34,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:41:34,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:41:34,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:41:34,514 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:41:34,515 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:41:34,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:41:34,516 DEBUG [Thread-1162 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:41:34,517 DEBUG [Thread-1159 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:41:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:41:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:41:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:41:34,518 DEBUG [Thread-1168 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:41:34,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:41:34,519 DEBUG [Thread-1168 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:41:34,519 DEBUG [Thread-1168 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:41:34,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:41:34,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:41:34,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:41:34,522 DEBUG [Thread-1157 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:41:34,523 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35876, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:41:34,524 DEBUG [Thread-1157 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:41:34,524 DEBUG [Thread-1157 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:41:34,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:41:34,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:41:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:34,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38de20de4b25bae04950a5d15dc346de 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T20:41:34,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=A 2024-11-26T20:41:34,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:34,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=B 2024-11-26T20:41:34,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:34,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=C 2024-11-26T20:41:34,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:34,528 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:41:34,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:41:34,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:41:34,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:41:34,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/96a8626f51de474890b5df0987c18261 is 50, key is test_row_0/A:col10/1732653694520/Put/seqid=0 2024-11-26T20:41:34,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653694564, max timestamp=1732653694573Num uniques 859; 2024-11-26T20:41:34,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:41:34,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653694564, max timestamp=1732653694574Num uniques 859; 2024-11-26T20:41:34,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:41:34,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653694564, max timestamp=1732653694574Num uniques 859; 2024-11-26T20:41:34,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:41:34,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T20:41:34,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653754605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:34,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:34,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:34,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653754607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:34,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:34,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653754608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:34,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653754609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:34,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:34,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:34,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653754610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:34,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:34,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742073_1249 (size=12001) 2024-11-26T20:41:34,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/96a8626f51de474890b5df0987c18261 2024-11-26T20:41:34,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T20:41:34,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:34,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:34,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:34,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:34,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:34,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:34,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/c6b48a1180c24e3db785c7b6cb17eb1d is 50, key is test_row_0/B:col10/1732653694520/Put/seqid=0 2024-11-26T20:41:34,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742074_1250 (size=12001) 2024-11-26T20:41:34,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653754714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:34,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:34,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653754724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:34,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653754724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653754724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:34,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653754724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:34,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T20:41:34,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:34,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:34,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:34,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:34,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T20:41:34,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653754924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:34,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:34,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653754934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:34,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:34,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653754934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:34,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:34,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653754934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:34,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653754934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:34,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T20:41:34,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:34,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:34,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:34,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:34,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:35,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T20:41:35,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:35,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:35,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:35,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:35,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:35,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/c6b48a1180c24e3db785c7b6cb17eb1d 2024-11-26T20:41:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T20:41:35,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/8db2a83c7b1c4200b4bc54c2be951818 is 50, key is test_row_0/C:col10/1732653694520/Put/seqid=0 2024-11-26T20:41:35,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742075_1251 (size=12001) 2024-11-26T20:41:35,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653755234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:35,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:35,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:35,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653755244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:35,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:35,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653755244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:35,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:35,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:35,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653755244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:35,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653755244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:35,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:35,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:35,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:35,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T20:41:35,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:35,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:35,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:35,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:35,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:35,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T20:41:35,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:35,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:35,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:35,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:35,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:35,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T20:41:35,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/8db2a83c7b1c4200b4bc54c2be951818 2024-11-26T20:41:35,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:35,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:35,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:35,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:35,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:35,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/96a8626f51de474890b5df0987c18261 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/96a8626f51de474890b5df0987c18261 2024-11-26T20:41:35,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/96a8626f51de474890b5df0987c18261, entries=150, sequenceid=11, filesize=11.7 K 2024-11-26T20:41:35,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/c6b48a1180c24e3db785c7b6cb17eb1d as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/c6b48a1180c24e3db785c7b6cb17eb1d 2024-11-26T20:41:35,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/c6b48a1180c24e3db785c7b6cb17eb1d, entries=150, sequenceid=11, filesize=11.7 K 2024-11-26T20:41:35,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/8db2a83c7b1c4200b4bc54c2be951818 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/8db2a83c7b1c4200b4bc54c2be951818 2024-11-26T20:41:35,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/8db2a83c7b1c4200b4bc54c2be951818, entries=150, sequenceid=11, filesize=11.7 K 2024-11-26T20:41:35,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 38de20de4b25bae04950a5d15dc346de in 1059ms, sequenceid=11, compaction requested=false 2024-11-26T20:41:35,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T20:41:35,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T20:41:35,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:35,711 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 38de20de4b25bae04950a5d15dc346de 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-26T20:41:35,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=A 2024-11-26T20:41:35,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:35,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=B 2024-11-26T20:41:35,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:35,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=C 2024-11-26T20:41:35,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:35,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/db24027e0558484881dc10249d5e5474 is 50, key is test_row_0/A:col10/1732653694609/Put/seqid=0 2024-11-26T20:41:35,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742076_1252 (size=12001) 2024-11-26T20:41:35,727 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/db24027e0558484881dc10249d5e5474 2024-11-26T20:41:35,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/14783aea350e489cb8e0d1b36cbb5892 is 50, key is test_row_0/B:col10/1732653694609/Put/seqid=0 2024-11-26T20:41:35,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742077_1253 (size=12001) 2024-11-26T20:41:35,743 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/14783aea350e489cb8e0d1b36cbb5892 2024-11-26T20:41:35,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:35,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653755758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:35,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:35,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653755758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:35,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:35,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653755758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:35,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:35,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653755760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:35,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:35,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:35,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:35,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653755760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:35,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:35,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:35,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/97eeb472ea7c46a0aee8adfe881206c2 is 50, key is test_row_0/C:col10/1732653694609/Put/seqid=0 2024-11-26T20:41:35,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742078_1254 (size=12001) 2024-11-26T20:41:35,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653755888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:35,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653755888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:35,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653755889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:35,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:35,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:35,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:35,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653755889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:35,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:35,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:35,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653755889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:35,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:35,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:36,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653756095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:36,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:36,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653756095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:36,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653756095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:36,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:36,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653756104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:36,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653756104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:36,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:36,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:36,181 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/97eeb472ea7c46a0aee8adfe881206c2 2024-11-26T20:41:36,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/db24027e0558484881dc10249d5e5474 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/db24027e0558484881dc10249d5e5474 2024-11-26T20:41:36,194 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/db24027e0558484881dc10249d5e5474, entries=150, sequenceid=48, filesize=11.7 K 2024-11-26T20:41:36,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/14783aea350e489cb8e0d1b36cbb5892 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/14783aea350e489cb8e0d1b36cbb5892 2024-11-26T20:41:36,202 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/14783aea350e489cb8e0d1b36cbb5892, entries=150, sequenceid=48, filesize=11.7 K 2024-11-26T20:41:36,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/97eeb472ea7c46a0aee8adfe881206c2 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/97eeb472ea7c46a0aee8adfe881206c2 2024-11-26T20:41:36,214 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/97eeb472ea7c46a0aee8adfe881206c2, entries=150, sequenceid=48, filesize=11.7 K 2024-11-26T20:41:36,215 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 38de20de4b25bae04950a5d15dc346de in 505ms, sequenceid=48, compaction requested=false 2024-11-26T20:41:36,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:36,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:36,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-26T20:41:36,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-26T20:41:36,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-26T20:41:36,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7380 sec 2024-11-26T20:41:36,225 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.7480 sec 2024-11-26T20:41:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:36,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38de20de4b25bae04950a5d15dc346de 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T20:41:36,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=A 2024-11-26T20:41:36,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:36,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=B 2024-11-26T20:41:36,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:36,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=C 2024-11-26T20:41:36,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:36,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/ea4e0750515d4cedbfed0ba0730e1fc8 is 50, key is test_row_0/A:col10/1732653696405/Put/seqid=0 2024-11-26T20:41:36,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653756437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:36,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:36,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653756439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:36,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653756445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:36,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:36,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:36,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653756447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653756446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:36,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:36,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742079_1255 (size=14341) 2024-11-26T20:41:36,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/ea4e0750515d4cedbfed0ba0730e1fc8 2024-11-26T20:41:36,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/ba9022b90e83479db422c950cfb600bb is 50, key is test_row_0/B:col10/1732653696405/Put/seqid=0 2024-11-26T20:41:36,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742080_1256 (size=12001) 2024-11-26T20:41:36,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653756554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:36,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653756557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:36,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653756564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653756564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:36,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653756565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T20:41:36,625 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:36,625 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:41:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-26T20:41:36,627 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:41:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T20:41:36,628 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:41:36,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:41:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T20:41:36,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653756764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:36,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653756764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:36,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:36,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653756774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:36,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:36,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653756774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:36,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:36,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653756774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:36,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:36,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:36,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:36,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T20:41:36,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:36,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:36,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:36,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:36,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:36,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/ba9022b90e83479db422c950cfb600bb 2024-11-26T20:41:36,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T20:41:36,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:36,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:36,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:36,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:36,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:36,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:36,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/327ce2783900477f8f1ebea081b34111 is 50, key is test_row_0/C:col10/1732653696405/Put/seqid=0 2024-11-26T20:41:36,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T20:41:36,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742081_1257 (size=12001) 2024-11-26T20:41:36,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/327ce2783900477f8f1ebea081b34111 2024-11-26T20:41:36,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/ea4e0750515d4cedbfed0ba0730e1fc8 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/ea4e0750515d4cedbfed0ba0730e1fc8 2024-11-26T20:41:36,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/ea4e0750515d4cedbfed0ba0730e1fc8, entries=200, sequenceid=61, filesize=14.0 K 2024-11-26T20:41:36,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/ba9022b90e83479db422c950cfb600bb as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/ba9022b90e83479db422c950cfb600bb 2024-11-26T20:41:36,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/ba9022b90e83479db422c950cfb600bb, entries=150, sequenceid=61, filesize=11.7 K 2024-11-26T20:41:36,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/327ce2783900477f8f1ebea081b34111 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/327ce2783900477f8f1ebea081b34111 2024-11-26T20:41:37,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/327ce2783900477f8f1ebea081b34111, entries=150, sequenceid=61, filesize=11.7 K 2024-11-26T20:41:37,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 38de20de4b25bae04950a5d15dc346de in 598ms, sequenceid=61, compaction requested=true 2024-11-26T20:41:37,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:37,004 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:37,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:41:37,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:37,005 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:37,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:41:37,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:37,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:41:37,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:37,005 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:37,005 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/A is initiating minor compaction (all files) 2024-11-26T20:41:37,005 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:37,006 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/A in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:37,006 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/B is initiating minor compaction (all files) 2024-11-26T20:41:37,006 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/B in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:37,006 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/96a8626f51de474890b5df0987c18261, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/db24027e0558484881dc10249d5e5474, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/ea4e0750515d4cedbfed0ba0730e1fc8] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=37.4 K 2024-11-26T20:41:37,006 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/c6b48a1180c24e3db785c7b6cb17eb1d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/14783aea350e489cb8e0d1b36cbb5892, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/ba9022b90e83479db422c950cfb600bb] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=35.2 K 2024-11-26T20:41:37,006 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting c6b48a1180c24e3db785c7b6cb17eb1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732653694511 2024-11-26T20:41:37,006 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96a8626f51de474890b5df0987c18261, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732653694511 2024-11-26T20:41:37,007 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting db24027e0558484881dc10249d5e5474, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732653694606 2024-11-26T20:41:37,007 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 14783aea350e489cb8e0d1b36cbb5892, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732653694606 2024-11-26T20:41:37,007 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea4e0750515d4cedbfed0ba0730e1fc8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732653695757 2024-11-26T20:41:37,007 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting ba9022b90e83479db422c950cfb600bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732653695757 2024-11-26T20:41:37,019 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#A#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:37,019 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/863039a46b584d69bdad265b20225dfa is 50, key is test_row_0/A:col10/1732653696405/Put/seqid=0 2024-11-26T20:41:37,023 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#B#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:37,023 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/eec6d554c0d1419a804fc66aeb57554c is 50, key is test_row_0/B:col10/1732653696405/Put/seqid=0 2024-11-26T20:41:37,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742082_1258 (size=12104) 2024-11-26T20:41:37,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742083_1259 (size=12104) 2024-11-26T20:41:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:37,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38de20de4b25bae04950a5d15dc346de 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-26T20:41:37,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=A 2024-11-26T20:41:37,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:37,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=B 2024-11-26T20:41:37,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:37,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=C 2024-11-26T20:41:37,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:37,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/f5987d5dc6ac4b369937dc185fa3e5bf is 50, key is test_row_0/A:col10/1732653697074/Put/seqid=0 2024-11-26T20:41:37,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T20:41:37,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:37,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:37,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:37,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:37,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:37,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653757091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:37,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:37,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653757092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:37,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653757093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:37,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:37,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:37,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742084_1260 (size=12001) 2024-11-26T20:41:37,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653757095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:37,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653757096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:37,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653757204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:37,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653757205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:37,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:37,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653757205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:37,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653757205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:37,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653757214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:37,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:37,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T20:41:37,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:37,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:37,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:37,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:37,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T20:41:37,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T20:41:37,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:37,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:37,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:37,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:37,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:37,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653757414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:37,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653757414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:37,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:37,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653757414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:37,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:37,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:37,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:37,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653757424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:37,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:37,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653757424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:37,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:37,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:37,452 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/863039a46b584d69bdad265b20225dfa as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/863039a46b584d69bdad265b20225dfa 2024-11-26T20:41:37,458 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/A of 38de20de4b25bae04950a5d15dc346de into 863039a46b584d69bdad265b20225dfa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:37,458 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:37,458 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/A, priority=13, startTime=1732653697004; duration=0sec 2024-11-26T20:41:37,458 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:37,458 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:A 2024-11-26T20:41:37,458 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:37,459 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:37,459 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/C is initiating minor compaction (all files) 2024-11-26T20:41:37,459 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/C in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:37,459 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/8db2a83c7b1c4200b4bc54c2be951818, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/97eeb472ea7c46a0aee8adfe881206c2, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/327ce2783900477f8f1ebea081b34111] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=35.2 K 2024-11-26T20:41:37,460 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8db2a83c7b1c4200b4bc54c2be951818, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732653694511 2024-11-26T20:41:37,460 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97eeb472ea7c46a0aee8adfe881206c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732653694606 2024-11-26T20:41:37,461 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 327ce2783900477f8f1ebea081b34111, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732653695757 2024-11-26T20:41:37,470 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#C#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:37,471 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/bb5006d640464ecd84ec31233f0b66ce is 50, key is test_row_0/C:col10/1732653696405/Put/seqid=0 2024-11-26T20:41:37,471 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/eec6d554c0d1419a804fc66aeb57554c as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/eec6d554c0d1419a804fc66aeb57554c 2024-11-26T20:41:37,477 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/B of 38de20de4b25bae04950a5d15dc346de into eec6d554c0d1419a804fc66aeb57554c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:37,477 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:37,477 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/B, priority=13, startTime=1732653697004; duration=0sec 2024-11-26T20:41:37,477 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:37,477 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:B 2024-11-26T20:41:37,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742085_1261 (size=12104) 2024-11-26T20:41:37,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/f5987d5dc6ac4b369937dc185fa3e5bf 2024-11-26T20:41:37,509 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T20:41:37,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/dfe06fa8f40241d692203333456474fd is 50, key is test_row_0/B:col10/1732653697074/Put/seqid=0 2024-11-26T20:41:37,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742086_1262 (size=12001) 2024-11-26T20:41:37,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T20:41:37,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:37,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:37,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:37,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:37,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:37,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:37,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T20:41:37,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:37,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:37,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:37,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:37,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:37,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:37,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:37,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653757724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:37,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:37,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:37,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653757724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:37,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:37,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653757734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:37,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:37,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653757734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:37,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653757734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:37,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:37,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T20:41:37,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T20:41:37,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:37,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:37,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:37,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:37,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:37,890 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/bb5006d640464ecd84ec31233f0b66ce as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/bb5006d640464ecd84ec31233f0b66ce 2024-11-26T20:41:37,896 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/C of 38de20de4b25bae04950a5d15dc346de into bb5006d640464ecd84ec31233f0b66ce(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:37,896 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:37,896 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/C, priority=13, startTime=1732653697005; duration=0sec 2024-11-26T20:41:37,896 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:37,896 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:C 2024-11-26T20:41:37,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/dfe06fa8f40241d692203333456474fd 2024-11-26T20:41:37,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/025dea5e84994973b487e38f3772d0eb is 50, key is test_row_0/C:col10/1732653697074/Put/seqid=0 2024-11-26T20:41:37,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742087_1263 (size=12001) 2024-11-26T20:41:37,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/025dea5e84994973b487e38f3772d0eb 2024-11-26T20:41:37,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/f5987d5dc6ac4b369937dc185fa3e5bf as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/f5987d5dc6ac4b369937dc185fa3e5bf 2024-11-26T20:41:37,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/f5987d5dc6ac4b369937dc185fa3e5bf, entries=150, sequenceid=82, filesize=11.7 K 2024-11-26T20:41:37,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/dfe06fa8f40241d692203333456474fd as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/dfe06fa8f40241d692203333456474fd 2024-11-26T20:41:37,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/dfe06fa8f40241d692203333456474fd, entries=150, sequenceid=82, filesize=11.7 K 2024-11-26T20:41:37,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/025dea5e84994973b487e38f3772d0eb as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/025dea5e84994973b487e38f3772d0eb 2024-11-26T20:41:37,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/025dea5e84994973b487e38f3772d0eb, entries=150, sequenceid=82, filesize=11.7 K 2024-11-26T20:41:37,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 38de20de4b25bae04950a5d15dc346de in 911ms, sequenceid=82, compaction requested=false 2024-11-26T20:41:37,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:38,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T20:41:38,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:38,019 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 38de20de4b25bae04950a5d15dc346de 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T20:41:38,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=A 2024-11-26T20:41:38,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:38,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=B 2024-11-26T20:41:38,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:38,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=C 2024-11-26T20:41:38,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:38,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/06b0f359ab574ccea96a191af14448cd is 50, key is test_row_0/A:col10/1732653697087/Put/seqid=0 2024-11-26T20:41:38,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742088_1264 (size=9657) 2024-11-26T20:41:38,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:38,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:38,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:38,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653758283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:38,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653698279, max timestamp=1732653698289Num uniques 859; 2024-11-26T20:41:38,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:41:38,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653698279, max timestamp=1732653698289Num uniques 859; 2024-11-26T20:41:38,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:41:38,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653698279, max timestamp=1732653698289Num uniques 859; 2024-11-26T20:41:38,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:41:38,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:38,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653758322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:38,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:38,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653758324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653758323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:38,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653758325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:38,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:38,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:38,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653758404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:38,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653758445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:38,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:38,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653758445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:38,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:38,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653758445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:38,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:38,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653758445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:38,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:38,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:38,481 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/06b0f359ab574ccea96a191af14448cd 2024-11-26T20:41:38,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/2895cfcd9e21489d9cdfde9843e2d885 is 50, key is test_row_0/B:col10/1732653697087/Put/seqid=0 2024-11-26T20:41:38,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742089_1265 (size=9657) 2024-11-26T20:41:38,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653758614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:38,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653758664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:38,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653758664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:38,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:38,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653758664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653758664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:38,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T20:41:38,904 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/2895cfcd9e21489d9cdfde9843e2d885 2024-11-26T20:41:38,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/4ccb0b4f8c8041588856622eb4d93cc2 is 50, key is test_row_0/C:col10/1732653697087/Put/seqid=0 2024-11-26T20:41:38,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742090_1266 (size=9657) 2024-11-26T20:41:38,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:38,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653758924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:38,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:38,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653758974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653758974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:38,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653758974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:38,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653758974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:39,320 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/4ccb0b4f8c8041588856622eb4d93cc2 2024-11-26T20:41:39,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/06b0f359ab574ccea96a191af14448cd as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/06b0f359ab574ccea96a191af14448cd 2024-11-26T20:41:39,329 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/06b0f359ab574ccea96a191af14448cd, entries=100, sequenceid=96, filesize=9.4 K 2024-11-26T20:41:39,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/2895cfcd9e21489d9cdfde9843e2d885 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/2895cfcd9e21489d9cdfde9843e2d885 2024-11-26T20:41:39,337 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/2895cfcd9e21489d9cdfde9843e2d885, entries=100, sequenceid=96, filesize=9.4 K 2024-11-26T20:41:39,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/4ccb0b4f8c8041588856622eb4d93cc2 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4ccb0b4f8c8041588856622eb4d93cc2 2024-11-26T20:41:39,342 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4ccb0b4f8c8041588856622eb4d93cc2, entries=100, sequenceid=96, filesize=9.4 K 2024-11-26T20:41:39,343 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 38de20de4b25bae04950a5d15dc346de in 1324ms, sequenceid=96, compaction requested=true 2024-11-26T20:41:39,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:39,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:39,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-26T20:41:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-26T20:41:39,347 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-26T20:41:39,348 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7170 sec 2024-11-26T20:41:39,350 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.7230 sec 2024-11-26T20:41:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:39,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38de20de4b25bae04950a5d15dc346de 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-26T20:41:39,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=A 2024-11-26T20:41:39,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:39,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=B 2024-11-26T20:41:39,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:39,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=C 2024-11-26T20:41:39,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:39,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/764e5666a89840beb1cbe77b784aa85c is 50, key is test_row_0/A:col10/1732653699446/Put/seqid=0 2024-11-26T20:41:39,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742091_1267 (size=14491) 2024-11-26T20:41:39,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653759484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:39,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653759484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:39,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653759484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:39,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:39,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:39,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:39,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653759484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:39,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:39,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:39,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:39,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653759486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:39,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:39,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:39,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:39,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:39,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653759594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:39,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:39,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:39,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:39,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653759804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:39,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:39,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:39,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:39,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/764e5666a89840beb1cbe77b784aa85c 2024-11-26T20:41:39,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/fcd4eddfdb094f15b02013c9253ccb21 is 50, key is test_row_0/B:col10/1732653699446/Put/seqid=0 2024-11-26T20:41:39,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742092_1268 (size=12101) 2024-11-26T20:41:40,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653760114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:40,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/fcd4eddfdb094f15b02013c9253ccb21 2024-11-26T20:41:40,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/8822bcf0802b4686b92eb46e61fccc59 is 50, key is test_row_0/C:col10/1732653699446/Put/seqid=0 2024-11-26T20:41:40,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742093_1269 (size=12101) 2024-11-26T20:41:40,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/8822bcf0802b4686b92eb46e61fccc59 2024-11-26T20:41:40,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/764e5666a89840beb1cbe77b784aa85c as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/764e5666a89840beb1cbe77b784aa85c 2024-11-26T20:41:40,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/764e5666a89840beb1cbe77b784aa85c, entries=200, sequenceid=132, filesize=14.2 K 2024-11-26T20:41:40,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/fcd4eddfdb094f15b02013c9253ccb21 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/fcd4eddfdb094f15b02013c9253ccb21 2024-11-26T20:41:40,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/fcd4eddfdb094f15b02013c9253ccb21, entries=150, sequenceid=132, filesize=11.8 K 2024-11-26T20:41:40,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/8822bcf0802b4686b92eb46e61fccc59 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/8822bcf0802b4686b92eb46e61fccc59 2024-11-26T20:41:40,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/8822bcf0802b4686b92eb46e61fccc59, entries=150, sequenceid=132, filesize=11.8 K 2024-11-26T20:41:40,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 38de20de4b25bae04950a5d15dc346de in 863ms, sequenceid=132, compaction requested=true 2024-11-26T20:41:40,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:40,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:41:40,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:40,311 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:41:40,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:41:40,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:40,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:41:40,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:40,311 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:41:40,313 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48253 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:41:40,313 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/A is initiating minor compaction (all files) 2024-11-26T20:41:40,313 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/A in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:40,313 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/863039a46b584d69bdad265b20225dfa, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/f5987d5dc6ac4b369937dc185fa3e5bf, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/06b0f359ab574ccea96a191af14448cd, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/764e5666a89840beb1cbe77b784aa85c] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=47.1 K 2024-11-26T20:41:40,313 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 863039a46b584d69bdad265b20225dfa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732653695757 2024-11-26T20:41:40,314 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5987d5dc6ac4b369937dc185fa3e5bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732653696436 2024-11-26T20:41:40,314 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06b0f359ab574ccea96a191af14448cd, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732653697087 2024-11-26T20:41:40,314 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45863 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:41:40,314 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/B is initiating minor compaction (all files) 2024-11-26T20:41:40,314 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/B in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:40,314 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 764e5666a89840beb1cbe77b784aa85c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732653698321 2024-11-26T20:41:40,314 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/eec6d554c0d1419a804fc66aeb57554c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/dfe06fa8f40241d692203333456474fd, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/2895cfcd9e21489d9cdfde9843e2d885, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/fcd4eddfdb094f15b02013c9253ccb21] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=44.8 K 2024-11-26T20:41:40,315 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting eec6d554c0d1419a804fc66aeb57554c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732653695757 2024-11-26T20:41:40,315 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting dfe06fa8f40241d692203333456474fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732653696436 2024-11-26T20:41:40,316 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 2895cfcd9e21489d9cdfde9843e2d885, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732653697087 2024-11-26T20:41:40,317 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting fcd4eddfdb094f15b02013c9253ccb21, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732653698321 2024-11-26T20:41:40,331 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#A#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:40,332 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/caec1d368b0942c281661589d882bb2b is 50, key is test_row_0/A:col10/1732653699446/Put/seqid=0 2024-11-26T20:41:40,335 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#B#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:40,335 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/71c7e7da2e01421cbaeb2d9b4da1f643 is 50, key is test_row_0/B:col10/1732653699446/Put/seqid=0 2024-11-26T20:41:40,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742094_1270 (size=12341) 2024-11-26T20:41:40,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742095_1271 (size=12341) 2024-11-26T20:41:40,351 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/71c7e7da2e01421cbaeb2d9b4da1f643 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/71c7e7da2e01421cbaeb2d9b4da1f643 2024-11-26T20:41:40,355 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/B of 38de20de4b25bae04950a5d15dc346de into 71c7e7da2e01421cbaeb2d9b4da1f643(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:40,356 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:40,356 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/B, priority=12, startTime=1732653700311; duration=0sec 2024-11-26T20:41:40,356 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:40,356 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:B 2024-11-26T20:41:40,356 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:41:40,357 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45863 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:41:40,357 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/C is initiating minor compaction (all files) 2024-11-26T20:41:40,357 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/C in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:40,357 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/bb5006d640464ecd84ec31233f0b66ce, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/025dea5e84994973b487e38f3772d0eb, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4ccb0b4f8c8041588856622eb4d93cc2, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/8822bcf0802b4686b92eb46e61fccc59] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=44.8 K 2024-11-26T20:41:40,358 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting bb5006d640464ecd84ec31233f0b66ce, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732653695757 2024-11-26T20:41:40,358 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 025dea5e84994973b487e38f3772d0eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732653696436 2024-11-26T20:41:40,359 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ccb0b4f8c8041588856622eb4d93cc2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732653697087 2024-11-26T20:41:40,360 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 8822bcf0802b4686b92eb46e61fccc59, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732653698321 2024-11-26T20:41:40,370 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#C#compaction#230 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:40,370 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/2da10ad447dc44b4aa4cf256e9f413be is 50, key is test_row_0/C:col10/1732653699446/Put/seqid=0 2024-11-26T20:41:40,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742096_1272 (size=12341) 2024-11-26T20:41:40,381 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/2da10ad447dc44b4aa4cf256e9f413be as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/2da10ad447dc44b4aa4cf256e9f413be 2024-11-26T20:41:40,387 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/C of 38de20de4b25bae04950a5d15dc346de into 2da10ad447dc44b4aa4cf256e9f413be(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:40,387 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:40,387 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/C, priority=12, startTime=1732653700312; duration=0sec 2024-11-26T20:41:40,387 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:40,388 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:C 2024-11-26T20:41:40,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:40,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38de20de4b25bae04950a5d15dc346de 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T20:41:40,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=A 2024-11-26T20:41:40,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:40,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=B 2024-11-26T20:41:40,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:40,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=C 2024-11-26T20:41:40,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:40,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/18af93be3a9b4c2f96e94a48879eb606 is 50, key is test_row_0/A:col10/1732653700496/Put/seqid=0 2024-11-26T20:41:40,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742097_1273 (size=14541) 2024-11-26T20:41:40,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:40,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653760525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:40,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:40,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732653700499, max timestamp=1732653700530Num uniques 859; 2024-11-26T20:41:40,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T20:41:40,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732653700499, max timestamp=1732653700530Num uniques 859; 2024-11-26T20:41:40,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T20:41:40,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653700523, max timestamp=1732653700530Num uniques 859; 2024-11-26T20:41:40,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:41:40,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:40,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653760559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:40,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:40,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:40,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653760561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:40,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:40,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653760561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:40,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:40,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:40,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:40,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653760624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:40,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:40,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653760634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:40,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:40,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653760674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653760674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653760674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:40,745 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/caec1d368b0942c281661589d882bb2b as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/caec1d368b0942c281661589d882bb2b 2024-11-26T20:41:40,750 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/A of 38de20de4b25bae04950a5d15dc346de into caec1d368b0942c281661589d882bb2b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:40,750 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:40,750 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/A, priority=12, startTime=1732653700311; duration=0sec 2024-11-26T20:41:40,750 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:40,750 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:A 2024-11-26T20:41:40,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T20:41:40,784 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:40,785 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:41:40,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-26T20:41:40,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T20:41:40,786 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:41:40,787 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:41:40,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:41:40,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:40,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653760843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:40,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:40,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:40,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:40,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653760884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:40,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653760884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:40,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653760884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:40,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:40,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T20:41:40,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/18af93be3a9b4c2f96e94a48879eb606 2024-11-26T20:41:40,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/0cd470c382e74b84a80b2f97d21ae197 is 50, key is test_row_0/B:col10/1732653700496/Put/seqid=0 2024-11-26T20:41:40,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742098_1274 (size=12151) 2024-11-26T20:41:40,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-26T20:41:40,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:40,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:40,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:40,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:40,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:41,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-26T20:41:41,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:41,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:41,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:41,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:41,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:41,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:41,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T20:41:41,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:41,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653761154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:41,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:41,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:41,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:41,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:41,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653761194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:41,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:41,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653761194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:41,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:41,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653761194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:41,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-26T20:41:41,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:41,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:41,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:41,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:41,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:41,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:41,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/0cd470c382e74b84a80b2f97d21ae197 2024-11-26T20:41:41,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/05cc7ce1a4b943319c18bd9832334fe2 is 50, key is test_row_0/C:col10/1732653700496/Put/seqid=0 2024-11-26T20:41:41,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742099_1275 (size=12151) 2024-11-26T20:41:41,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-26T20:41:41,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:41,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:41,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:41,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:41,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T20:41:41,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-26T20:41:41,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:41,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:41,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:41,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:41,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:41,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653761644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:41,646 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_2' in 38de20de4b25bae04950a5d15dc346de of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:41:41,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:41,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:41,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:41,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:41,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653761664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:41,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-26T20:41:41,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:41,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:41,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:41,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:41,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:41,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:41,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653761704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:41,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:41,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:41,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:41,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:41,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653761704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:41,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:41,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653761714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:41,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/05cc7ce1a4b943319c18bd9832334fe2 2024-11-26T20:41:41,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/18af93be3a9b4c2f96e94a48879eb606 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/18af93be3a9b4c2f96e94a48879eb606 2024-11-26T20:41:41,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/18af93be3a9b4c2f96e94a48879eb606, entries=200, sequenceid=146, filesize=14.2 K 2024-11-26T20:41:41,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/0cd470c382e74b84a80b2f97d21ae197 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/0cd470c382e74b84a80b2f97d21ae197 2024-11-26T20:41:41,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/0cd470c382e74b84a80b2f97d21ae197, entries=150, sequenceid=146, filesize=11.9 K 2024-11-26T20:41:41,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/05cc7ce1a4b943319c18bd9832334fe2 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/05cc7ce1a4b943319c18bd9832334fe2 2024-11-26T20:41:41,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/05cc7ce1a4b943319c18bd9832334fe2, entries=150, sequenceid=146, filesize=11.9 K 2024-11-26T20:41:41,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=123.00 KB/125950 for 38de20de4b25bae04950a5d15dc346de in 1273ms, sequenceid=146, compaction requested=false 2024-11-26T20:41:41,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:41,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-26T20:41:41,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:41,861 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 38de20de4b25bae04950a5d15dc346de 3/3 column families, dataSize=123.00 KB heapSize=345.53 KB 2024-11-26T20:41:41,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=A 2024-11-26T20:41:41,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:41,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=B 2024-11-26T20:41:41,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:41,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=C 2024-11-26T20:41:41,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:41,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/1c1479398fab4aa7ae02f5bd47c866e4 is 50, key is test_row_0/A:col10/1732653700560/Put/seqid=0 2024-11-26T20:41:41,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742100_1276 (size=12151) 2024-11-26T20:41:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T20:41:42,271 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/1c1479398fab4aa7ae02f5bd47c866e4 2024-11-26T20:41:42,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/733aba2add344a4cb7757faf8caf5e87 is 50, key is test_row_0/B:col10/1732653700560/Put/seqid=0 2024-11-26T20:41:42,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742101_1277 (size=12151) 2024-11-26T20:41:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:42,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:42,686 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/733aba2add344a4cb7757faf8caf5e87 2024-11-26T20:41:42,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/a40ed1fba02242f5ad0fd903ed1bea02 is 50, key is test_row_0/C:col10/1732653700560/Put/seqid=0 2024-11-26T20:41:42,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742102_1278 (size=12151) 2024-11-26T20:41:42,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653762724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:42,728 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 38de20de4b25bae04950a5d15dc346de of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:41:42,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:42,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:42,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:42,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653762726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:42,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653762727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:42,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653762734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:42,740 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 38de20de4b25bae04950a5d15dc346de of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:41:42,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:42,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:42,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:42,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:42,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653762844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:42,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:42,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:42,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:42,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:42,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653762844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:42,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:42,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:42,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T20:41:43,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653763054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:43,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653763054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:43,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:43,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:43,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:43,110 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/a40ed1fba02242f5ad0fd903ed1bea02 2024-11-26T20:41:43,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/1c1479398fab4aa7ae02f5bd47c866e4 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/1c1479398fab4aa7ae02f5bd47c866e4 2024-11-26T20:41:43,118 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/1c1479398fab4aa7ae02f5bd47c866e4, entries=150, sequenceid=181, filesize=11.9 K 2024-11-26T20:41:43,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/733aba2add344a4cb7757faf8caf5e87 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/733aba2add344a4cb7757faf8caf5e87 2024-11-26T20:41:43,123 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/733aba2add344a4cb7757faf8caf5e87, entries=150, sequenceid=181, filesize=11.9 K 2024-11-26T20:41:43,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/a40ed1fba02242f5ad0fd903ed1bea02 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/a40ed1fba02242f5ad0fd903ed1bea02 2024-11-26T20:41:43,128 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/a40ed1fba02242f5ad0fd903ed1bea02, entries=150, sequenceid=181, filesize=11.9 K 2024-11-26T20:41:43,128 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~123.00 KB/125950, heapSize ~345.48 KB/353776, currentSize=60.38 KB/61830 for 38de20de4b25bae04950a5d15dc346de in 1267ms, sequenceid=181, compaction requested=true 2024-11-26T20:41:43,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:43,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:43,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-26T20:41:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-26T20:41:43,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-26T20:41:43,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3460 sec 2024-11-26T20:41:43,137 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.3510 sec 2024-11-26T20:41:43,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:43,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38de20de4b25bae04950a5d15dc346de 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T20:41:43,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=A 2024-11-26T20:41:43,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:43,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=B 2024-11-26T20:41:43,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:43,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=C 2024-11-26T20:41:43,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:43,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/e2ad7981d9224a1898b7322c0aaac5cf is 50, key is test_row_0/A:col10/1732653703365/Put/seqid=0 2024-11-26T20:41:43,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742103_1279 (size=14541) 2024-11-26T20:41:43,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/e2ad7981d9224a1898b7322c0aaac5cf 2024-11-26T20:41:43,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/02137ae1bfb340709d97a013df9bfb7a is 50, key is test_row_0/B:col10/1732653703365/Put/seqid=0 2024-11-26T20:41:43,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742104_1280 (size=12151) 2024-11-26T20:41:43,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:43,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653763412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:43,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:43,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:43,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:43,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:43,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653763415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:43,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:43,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:43,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:43,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653763524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:43,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653763524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:43,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653763664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:43,669 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_2' in 38de20de4b25bae04950a5d15dc346de of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:41:43,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:43,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:43,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:43,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653763734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:43,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:43,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653763734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:43,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:43,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:43,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:43,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:43,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:43,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/02137ae1bfb340709d97a013df9bfb7a 2024-11-26T20:41:43,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/4a159b94b29d4711a0b9ba3b6bf97678 is 50, key is test_row_0/C:col10/1732653703365/Put/seqid=0 2024-11-26T20:41:43,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742105_1281 (size=12151) 2024-11-26T20:41:44,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653764044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:44,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653764044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:44,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:44,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:44,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:44,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/4a159b94b29d4711a0b9ba3b6bf97678 2024-11-26T20:41:44,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/e2ad7981d9224a1898b7322c0aaac5cf as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e2ad7981d9224a1898b7322c0aaac5cf 2024-11-26T20:41:44,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e2ad7981d9224a1898b7322c0aaac5cf, entries=200, sequenceid=195, filesize=14.2 K 2024-11-26T20:41:44,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/02137ae1bfb340709d97a013df9bfb7a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/02137ae1bfb340709d97a013df9bfb7a 2024-11-26T20:41:44,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/02137ae1bfb340709d97a013df9bfb7a, entries=150, sequenceid=195, filesize=11.9 K 2024-11-26T20:41:44,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/4a159b94b29d4711a0b9ba3b6bf97678 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4a159b94b29d4711a0b9ba3b6bf97678 2024-11-26T20:41:44,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4a159b94b29d4711a0b9ba3b6bf97678, entries=150, sequenceid=195, filesize=11.9 K 2024-11-26T20:41:44,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 38de20de4b25bae04950a5d15dc346de in 865ms, sequenceid=195, compaction requested=true 2024-11-26T20:41:44,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:44,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:41:44,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:44,233 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:41:44,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:41:44,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:44,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:41:44,233 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:41:44,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:44,234 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:41:44,234 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53574 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:41:44,234 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/B is initiating minor compaction (all files) 2024-11-26T20:41:44,234 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/A is initiating minor compaction (all files) 2024-11-26T20:41:44,234 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/B in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:44,234 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/A in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:44,234 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/71c7e7da2e01421cbaeb2d9b4da1f643, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/0cd470c382e74b84a80b2f97d21ae197, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/733aba2add344a4cb7757faf8caf5e87, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/02137ae1bfb340709d97a013df9bfb7a] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=47.7 K 2024-11-26T20:41:44,234 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/caec1d368b0942c281661589d882bb2b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/18af93be3a9b4c2f96e94a48879eb606, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/1c1479398fab4aa7ae02f5bd47c866e4, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e2ad7981d9224a1898b7322c0aaac5cf] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=52.3 K 2024-11-26T20:41:44,235 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting caec1d368b0942c281661589d882bb2b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732653698321 2024-11-26T20:41:44,235 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 71c7e7da2e01421cbaeb2d9b4da1f643, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732653698321 2024-11-26T20:41:44,235 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18af93be3a9b4c2f96e94a48879eb606, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732653699457 2024-11-26T20:41:44,235 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cd470c382e74b84a80b2f97d21ae197, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732653699457 2024-11-26T20:41:44,235 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c1479398fab4aa7ae02f5bd47c866e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732653700552 2024-11-26T20:41:44,235 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 733aba2add344a4cb7757faf8caf5e87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732653700552 2024-11-26T20:41:44,236 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2ad7981d9224a1898b7322c0aaac5cf, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732653702719 2024-11-26T20:41:44,236 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 02137ae1bfb340709d97a013df9bfb7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732653702719 2024-11-26T20:41:44,248 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#B#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:44,249 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#A#compaction#240 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:44,249 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/31cad0aa5ceb45919d24d464ef67419a is 50, key is test_row_0/A:col10/1732653703365/Put/seqid=0 2024-11-26T20:41:44,251 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/73265d5c2a4841efb78a9dca1479f831 is 50, key is test_row_0/B:col10/1732653703365/Put/seqid=0 2024-11-26T20:41:44,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742106_1282 (size=12527) 2024-11-26T20:41:44,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742107_1283 (size=12527) 2024-11-26T20:41:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:44,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38de20de4b25bae04950a5d15dc346de 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T20:41:44,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=A 2024-11-26T20:41:44,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:44,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=B 2024-11-26T20:41:44,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:44,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=C 2024-11-26T20:41:44,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:44,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/c13d7586681a4847ba2dd01bf8d897ab is 50, key is test_row_0/A:col10/1732653703414/Put/seqid=0 2024-11-26T20:41:44,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742108_1284 (size=14541) 2024-11-26T20:41:44,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653764580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:44,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653764585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:44,660 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/31cad0aa5ceb45919d24d464ef67419a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/31cad0aa5ceb45919d24d464ef67419a 2024-11-26T20:41:44,664 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/A of 38de20de4b25bae04950a5d15dc346de into 31cad0aa5ceb45919d24d464ef67419a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:44,664 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:44,664 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/A, priority=12, startTime=1732653704232; duration=0sec 2024-11-26T20:41:44,665 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:44,665 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:A 2024-11-26T20:41:44,665 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:41:44,669 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:41:44,669 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/C is initiating minor compaction (all files) 2024-11-26T20:41:44,669 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/C in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:44,669 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/2da10ad447dc44b4aa4cf256e9f413be, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/05cc7ce1a4b943319c18bd9832334fe2, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/a40ed1fba02242f5ad0fd903ed1bea02, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4a159b94b29d4711a0b9ba3b6bf97678] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=47.7 K 2024-11-26T20:41:44,670 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2da10ad447dc44b4aa4cf256e9f413be, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732653698321 2024-11-26T20:41:44,670 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05cc7ce1a4b943319c18bd9832334fe2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732653699457 2024-11-26T20:41:44,671 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting a40ed1fba02242f5ad0fd903ed1bea02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732653700552 2024-11-26T20:41:44,671 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/73265d5c2a4841efb78a9dca1479f831 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/73265d5c2a4841efb78a9dca1479f831 2024-11-26T20:41:44,671 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a159b94b29d4711a0b9ba3b6bf97678, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732653702719 2024-11-26T20:41:44,676 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/B of 38de20de4b25bae04950a5d15dc346de into 73265d5c2a4841efb78a9dca1479f831(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:44,676 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:44,676 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/B, priority=12, startTime=1732653704233; duration=0sec 2024-11-26T20:41:44,676 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:44,676 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:B 2024-11-26T20:41:44,681 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#C#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:44,682 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/469586fd06bc4a15ba2e1168d504d902 is 50, key is test_row_0/C:col10/1732653703365/Put/seqid=0 2024-11-26T20:41:44,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742109_1285 (size=12527) 2024-11-26T20:41:44,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653764694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:44,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653764694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:44,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653764743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:44,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653764744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:44,745 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 38de20de4b25bae04950a5d15dc346de of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:41:44,746 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 38de20de4b25bae04950a5d15dc346de of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:41:44,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:44,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:44,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:44,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:44,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:44,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:44,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:44,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653764904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:44,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:44,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653764904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:44,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T20:41:44,945 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:44,945 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:41:44,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-26T20:41:44,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T20:41:44,947 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:41:44,947 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:41:44,948 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:41:44,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/c13d7586681a4847ba2dd01bf8d897ab 2024-11-26T20:41:44,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/9b74add8d2494e33872a17d215d03eb2 is 50, key is test_row_0/B:col10/1732653703414/Put/seqid=0 2024-11-26T20:41:44,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742110_1286 (size=12151) 2024-11-26T20:41:45,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T20:41:45,091 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/469586fd06bc4a15ba2e1168d504d902 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/469586fd06bc4a15ba2e1168d504d902 2024-11-26T20:41:45,095 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/C of 38de20de4b25bae04950a5d15dc346de into 469586fd06bc4a15ba2e1168d504d902(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:45,095 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:45,096 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/C, priority=12, startTime=1732653704233; duration=0sec 2024-11-26T20:41:45,096 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:45,096 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:C 2024-11-26T20:41:45,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T20:41:45,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:45,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:45,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:45,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:45,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:45,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:45,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:45,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:45,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653765213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:45,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653765213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:45,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T20:41:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:45,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:45,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T20:41:45,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/9b74add8d2494e33872a17d215d03eb2 2024-11-26T20:41:45,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/98e055d6b38941f4b02544c9aab7e26d is 50, key is test_row_0/C:col10/1732653703414/Put/seqid=0 2024-11-26T20:41:45,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742111_1287 (size=12151) 2024-11-26T20:41:45,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T20:41:45,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:45,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:45,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:45,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:45,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:45,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T20:41:45,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:45,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:45,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:45,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:45,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:45,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T20:41:45,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T20:41:45,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:45,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:45,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:45,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:45,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:45,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653765723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:45,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653765724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:45,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:45,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:45,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:45,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:45,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:45,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:45,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/98e055d6b38941f4b02544c9aab7e26d 2024-11-26T20:41:45,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/c13d7586681a4847ba2dd01bf8d897ab as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/c13d7586681a4847ba2dd01bf8d897ab 2024-11-26T20:41:45,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/c13d7586681a4847ba2dd01bf8d897ab, entries=200, sequenceid=215, filesize=14.2 K 2024-11-26T20:41:45,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/9b74add8d2494e33872a17d215d03eb2 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/9b74add8d2494e33872a17d215d03eb2 2024-11-26T20:41:45,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/9b74add8d2494e33872a17d215d03eb2, entries=150, sequenceid=215, filesize=11.9 K 2024-11-26T20:41:45,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/98e055d6b38941f4b02544c9aab7e26d as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/98e055d6b38941f4b02544c9aab7e26d 2024-11-26T20:41:45,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/98e055d6b38941f4b02544c9aab7e26d, entries=150, sequenceid=215, filesize=11.9 K 2024-11-26T20:41:45,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 38de20de4b25bae04950a5d15dc346de in 1261ms, sequenceid=215, compaction requested=false 2024-11-26T20:41:45,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:45,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T20:41:45,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:45,865 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 38de20de4b25bae04950a5d15dc346de 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T20:41:45,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=A 2024-11-26T20:41:45,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:45,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=B 2024-11-26T20:41:45,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:45,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=C 2024-11-26T20:41:45,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:45,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/56eb7709a5bd4b5ea55a019c75a8895a is 50, key is test_row_0/A:col10/1732653704578/Put/seqid=0 2024-11-26T20:41:45,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742112_1288 (size=12151) 2024-11-26T20:41:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T20:41:46,275 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/56eb7709a5bd4b5ea55a019c75a8895a 2024-11-26T20:41:46,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/13d26e25174647fcb159c7e078c67ba1 is 50, key is test_row_0/B:col10/1732653704578/Put/seqid=0 2024-11-26T20:41:46,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742113_1289 (size=12151) 2024-11-26T20:41:46,286 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/13d26e25174647fcb159c7e078c67ba1 2024-11-26T20:41:46,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/4446765b0e7c441ca03e5d84efe4204e is 50, key is test_row_0/C:col10/1732653704578/Put/seqid=0 2024-11-26T20:41:46,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742114_1290 (size=12151) 2024-11-26T20:41:46,699 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/4446765b0e7c441ca03e5d84efe4204e 2024-11-26T20:41:46,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/56eb7709a5bd4b5ea55a019c75a8895a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/56eb7709a5bd4b5ea55a019c75a8895a 2024-11-26T20:41:46,707 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/56eb7709a5bd4b5ea55a019c75a8895a, entries=150, sequenceid=230, filesize=11.9 K 2024-11-26T20:41:46,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/13d26e25174647fcb159c7e078c67ba1 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/13d26e25174647fcb159c7e078c67ba1 2024-11-26T20:41:46,711 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/13d26e25174647fcb159c7e078c67ba1, entries=150, sequenceid=230, filesize=11.9 K 2024-11-26T20:41:46,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/4446765b0e7c441ca03e5d84efe4204e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4446765b0e7c441ca03e5d84efe4204e 2024-11-26T20:41:46,716 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4446765b0e7c441ca03e5d84efe4204e, entries=150, sequenceid=230, filesize=11.9 K 2024-11-26T20:41:46,717 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 38de20de4b25bae04950a5d15dc346de in 852ms, sequenceid=230, compaction requested=true 2024-11-26T20:41:46,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:46,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:46,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-26T20:41:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-26T20:41:46,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-26T20:41:46,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7720 sec 2024-11-26T20:41:46,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.7760 sec 2024-11-26T20:41:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:46,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38de20de4b25bae04950a5d15dc346de 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T20:41:46,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=A 2024-11-26T20:41:46,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:46,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=B 2024-11-26T20:41:46,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:46,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=C 2024-11-26T20:41:46,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:46,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/094fb5ae219b4bd5af4ff07fcd6f2a3b is 50, key is test_row_0/A:col10/1732653706757/Put/seqid=0 2024-11-26T20:41:46,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742115_1291 (size=9757) 2024-11-26T20:41:46,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653706774, max timestamp=1732653706803Num uniques 859; 2024-11-26T20:41:46,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:41:46,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653706774, max timestamp=1732653706803Num uniques 859; 2024-11-26T20:41:46,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653706774, max timestamp=1732653706803Num uniques 859; 2024-11-26T20:41:46,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:41:46,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:41:46,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653766840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:46,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:46,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653766841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:46,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:46,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653766954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:46,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:46,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653766954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:47,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T20:41:47,094 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:47,095 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:41:47,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-26T20:41:47,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T20:41:47,097 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:41:47,098 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:41:47,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:41:47,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/094fb5ae219b4bd5af4ff07fcd6f2a3b 2024-11-26T20:41:47,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653767164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:47,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:47,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:47,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:47,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653767164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:47,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/a5624676890c44f7af408ed5b131ae0f is 50, key is test_row_0/B:col10/1732653706757/Put/seqid=0 2024-11-26T20:41:47,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742116_1292 (size=9757) 2024-11-26T20:41:47,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T20:41:47,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T20:41:47,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:47,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:47,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:47,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:47,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:47,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:47,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T20:41:47,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:47,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:47,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:47,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:47,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:47,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T20:41:47,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653767473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:47,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:47,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:47,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:47,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653767473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:47,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:47,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:47,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:47,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T20:41:47,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:47,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:47,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:47,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:47,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:47,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/a5624676890c44f7af408ed5b131ae0f 2024-11-26T20:41:47,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/6206143a93954b90aaa935d19880d2ca is 50, key is test_row_0/C:col10/1732653706757/Put/seqid=0 2024-11-26T20:41:47,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742117_1293 (size=9757) 2024-11-26T20:41:47,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35822 deadline: 1732653767674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:47,678 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_2' in 38de20de4b25bae04950a5d15dc346de of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:41:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:47,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T20:41:47,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:47,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:47,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:47,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:47,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T20:41:47,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T20:41:47,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:47,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:47,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:47,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:47,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:47,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:47,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:47,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653767983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:47,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:47,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653767983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:47,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:47,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:47,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:48,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/6206143a93954b90aaa935d19880d2ca 2024-11-26T20:41:48,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/094fb5ae219b4bd5af4ff07fcd6f2a3b as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/094fb5ae219b4bd5af4ff07fcd6f2a3b 2024-11-26T20:41:48,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/094fb5ae219b4bd5af4ff07fcd6f2a3b, entries=100, sequenceid=240, filesize=9.5 K 2024-11-26T20:41:48,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/a5624676890c44f7af408ed5b131ae0f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/a5624676890c44f7af408ed5b131ae0f 2024-11-26T20:41:48,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/a5624676890c44f7af408ed5b131ae0f, entries=100, sequenceid=240, filesize=9.5 K 2024-11-26T20:41:48,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/6206143a93954b90aaa935d19880d2ca as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/6206143a93954b90aaa935d19880d2ca 2024-11-26T20:41:48,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T20:41:48,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:48,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:48,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:48,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:48,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:48,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/6206143a93954b90aaa935d19880d2ca, entries=100, sequenceid=240, filesize=9.5 K 2024-11-26T20:41:48,019 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 38de20de4b25bae04950a5d15dc346de in 1261ms, sequenceid=240, compaction requested=true 2024-11-26T20:41:48,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:48,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:41:48,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:48,019 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:41:48,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:41:48,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:48,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:41:48,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T20:41:48,019 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:41:48,020 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48976 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:41:48,020 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/A is initiating minor compaction (all files) 2024-11-26T20:41:48,020 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/A in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:48,020 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46586 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:41:48,020 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/B is initiating minor compaction (all files) 2024-11-26T20:41:48,020 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/31cad0aa5ceb45919d24d464ef67419a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/c13d7586681a4847ba2dd01bf8d897ab, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/56eb7709a5bd4b5ea55a019c75a8895a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/094fb5ae219b4bd5af4ff07fcd6f2a3b] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=47.8 K 2024-11-26T20:41:48,020 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/B in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:48,020 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/73265d5c2a4841efb78a9dca1479f831, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/9b74add8d2494e33872a17d215d03eb2, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/13d26e25174647fcb159c7e078c67ba1, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/a5624676890c44f7af408ed5b131ae0f] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=45.5 K 2024-11-26T20:41:48,021 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 31cad0aa5ceb45919d24d464ef67419a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732653702719 2024-11-26T20:41:48,021 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73265d5c2a4841efb78a9dca1479f831, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732653702719 2024-11-26T20:41:48,021 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting c13d7586681a4847ba2dd01bf8d897ab, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732653703410 2024-11-26T20:41:48,021 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 56eb7709a5bd4b5ea55a019c75a8895a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732653704570 2024-11-26T20:41:48,021 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b74add8d2494e33872a17d215d03eb2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732653703410 2024-11-26T20:41:48,022 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 094fb5ae219b4bd5af4ff07fcd6f2a3b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732653706754 2024-11-26T20:41:48,022 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13d26e25174647fcb159c7e078c67ba1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732653704570 2024-11-26T20:41:48,022 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5624676890c44f7af408ed5b131ae0f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732653706754 2024-11-26T20:41:48,032 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#A#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:48,032 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/be2a2106329943e186b1a4727c290b9a is 50, key is test_row_0/A:col10/1732653706757/Put/seqid=0 2024-11-26T20:41:48,035 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#B#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:48,035 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/dcdbb8563fcb4077a40e4ff0456f3a6d is 50, key is test_row_0/B:col10/1732653706757/Put/seqid=0 2024-11-26T20:41:48,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742119_1295 (size=12663) 2024-11-26T20:41:48,052 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/dcdbb8563fcb4077a40e4ff0456f3a6d as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/dcdbb8563fcb4077a40e4ff0456f3a6d 2024-11-26T20:41:48,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742118_1294 (size=12663) 2024-11-26T20:41:48,059 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/B of 38de20de4b25bae04950a5d15dc346de into dcdbb8563fcb4077a40e4ff0456f3a6d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:48,059 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:48,059 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/B, priority=12, startTime=1732653708019; duration=0sec 2024-11-26T20:41:48,059 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:48,059 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:B 2024-11-26T20:41:48,059 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:41:48,060 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46586 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:41:48,061 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/C is initiating minor compaction (all files) 2024-11-26T20:41:48,061 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/C in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:48,061 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/469586fd06bc4a15ba2e1168d504d902, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/98e055d6b38941f4b02544c9aab7e26d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4446765b0e7c441ca03e5d84efe4204e, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/6206143a93954b90aaa935d19880d2ca] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=45.5 K 2024-11-26T20:41:48,061 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 469586fd06bc4a15ba2e1168d504d902, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732653702719 2024-11-26T20:41:48,062 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98e055d6b38941f4b02544c9aab7e26d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732653703410 2024-11-26T20:41:48,062 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4446765b0e7c441ca03e5d84efe4204e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732653704570 2024-11-26T20:41:48,063 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6206143a93954b90aaa935d19880d2ca, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732653706754 2024-11-26T20:41:48,066 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/be2a2106329943e186b1a4727c290b9a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/be2a2106329943e186b1a4727c290b9a 2024-11-26T20:41:48,071 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/A of 38de20de4b25bae04950a5d15dc346de into be2a2106329943e186b1a4727c290b9a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:48,071 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:48,071 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/A, priority=12, startTime=1732653708019; duration=0sec 2024-11-26T20:41:48,071 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:48,071 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:A 2024-11-26T20:41:48,072 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#C#compaction#254 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:48,072 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/d89b88f20b1e43c7802ca4f81b3c3ccd is 50, key is test_row_0/C:col10/1732653706757/Put/seqid=0 2024-11-26T20:41:48,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742120_1296 (size=12663) 2024-11-26T20:41:48,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T20:41:48,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:48,169 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 38de20de4b25bae04950a5d15dc346de 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-26T20:41:48,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=A 2024-11-26T20:41:48,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:48,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=B 2024-11-26T20:41:48,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:48,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=C 2024-11-26T20:41:48,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:48,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/12a334717bf44a679037b32c49c1fabf is 50, key is test_row_0/A:col10/1732653706832/Put/seqid=0 2024-11-26T20:41:48,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742121_1297 (size=12301) 2024-11-26T20:41:48,184 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/12a334717bf44a679037b32c49c1fabf 2024-11-26T20:41:48,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/2ed5f5e0b924418e8912793c6918557a is 50, key is test_row_0/B:col10/1732653706832/Put/seqid=0 2024-11-26T20:41:48,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742122_1298 (size=12301) 2024-11-26T20:41:48,196 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/2ed5f5e0b924418e8912793c6918557a 2024-11-26T20:41:48,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/00f21f1315ab416088ae6e639bda6c3d is 50, key is test_row_0/C:col10/1732653706832/Put/seqid=0 2024-11-26T20:41:48,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742123_1299 (size=12301) 2024-11-26T20:41:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T20:41:48,480 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/d89b88f20b1e43c7802ca4f81b3c3ccd as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/d89b88f20b1e43c7802ca4f81b3c3ccd 2024-11-26T20:41:48,486 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/C of 38de20de4b25bae04950a5d15dc346de into d89b88f20b1e43c7802ca4f81b3c3ccd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:48,486 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:48,486 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/C, priority=12, startTime=1732653708019; duration=0sec 2024-11-26T20:41:48,486 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:48,486 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:C 2024-11-26T20:41:48,608 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/00f21f1315ab416088ae6e639bda6c3d 2024-11-26T20:41:48,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/12a334717bf44a679037b32c49c1fabf as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/12a334717bf44a679037b32c49c1fabf 2024-11-26T20:41:48,617 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/12a334717bf44a679037b32c49c1fabf, entries=150, sequenceid=278, filesize=12.0 K 2024-11-26T20:41:48,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/2ed5f5e0b924418e8912793c6918557a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/2ed5f5e0b924418e8912793c6918557a 2024-11-26T20:41:48,622 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/2ed5f5e0b924418e8912793c6918557a, entries=150, sequenceid=278, filesize=12.0 K 2024-11-26T20:41:48,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/00f21f1315ab416088ae6e639bda6c3d as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/00f21f1315ab416088ae6e639bda6c3d 2024-11-26T20:41:48,626 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/00f21f1315ab416088ae6e639bda6c3d, entries=150, sequenceid=278, filesize=12.0 K 2024-11-26T20:41:48,627 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=0 B/0 for 38de20de4b25bae04950a5d15dc346de in 458ms, sequenceid=278, compaction requested=false 2024-11-26T20:41:48,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:48,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:48,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-26T20:41:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-26T20:41:48,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-26T20:41:48,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5320 sec 2024-11-26T20:41:48,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.5390 sec 2024-11-26T20:41:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:48,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38de20de4b25bae04950a5d15dc346de 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T20:41:48,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=A 2024-11-26T20:41:48,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:48,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=B 2024-11-26T20:41:48,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:48,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=C 2024-11-26T20:41:48,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:48,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/b95789dee17a45dab717eb0a3b68098f is 50, key is test_row_0/A:col10/1732653708756/Put/seqid=0 2024-11-26T20:41:48,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742124_1300 (size=14741) 2024-11-26T20:41:48,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653708825, max timestamp=1732653708859Num uniques 859; 2024-11-26T20:41:48,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653708825, max timestamp=1732653708859Num uniques 859; 2024-11-26T20:41:48,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653708825, max timestamp=1732653708859Num uniques 859; 2024-11-26T20:41:48,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:41:48,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:41:48,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:41:48,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653768908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:48,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653768912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:49,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:49,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653769004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:49,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:49,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653769004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:49,010 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_1' in 38de20de4b25bae04950a5d15dc346de of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:41:49,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:49,011 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 38de20de4b25bae04950a5d15dc346de of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:41:49,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:49,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653769024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:49,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653769024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:49,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:49,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/b95789dee17a45dab717eb0a3b68098f 2024-11-26T20:41:49,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/37b7e851fb9f4a8687d4e4691454487c is 50, key is test_row_0/B:col10/1732653708756/Put/seqid=0 2024-11-26T20:41:49,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742125_1301 (size=12301) 2024-11-26T20:41:49,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/37b7e851fb9f4a8687d4e4691454487c 2024-11-26T20:41:49,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/c700e6b4248d49a0b00294236d1a7750 is 50, key is test_row_0/C:col10/1732653708756/Put/seqid=0 2024-11-26T20:41:49,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742126_1302 (size=12301) 2024-11-26T20:41:49,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/c700e6b4248d49a0b00294236d1a7750 2024-11-26T20:41:49,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/b95789dee17a45dab717eb0a3b68098f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/b95789dee17a45dab717eb0a3b68098f 2024-11-26T20:41:49,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:49,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653769233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:49,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/b95789dee17a45dab717eb0a3b68098f, entries=200, sequenceid=289, filesize=14.4 K 2024-11-26T20:41:49,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653769235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:49,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/37b7e851fb9f4a8687d4e4691454487c as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/37b7e851fb9f4a8687d4e4691454487c 2024-11-26T20:41:49,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/37b7e851fb9f4a8687d4e4691454487c, entries=150, sequenceid=289, filesize=12.0 K 2024-11-26T20:41:49,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/c700e6b4248d49a0b00294236d1a7750 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/c700e6b4248d49a0b00294236d1a7750 2024-11-26T20:41:49,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T20:41:49,244 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:49,245 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:41:49,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-26T20:41:49,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T20:41:49,247 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:41:49,248 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:41:49,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:41:49,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/c700e6b4248d49a0b00294236d1a7750, entries=150, sequenceid=289, filesize=12.0 K 2024-11-26T20:41:49,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 38de20de4b25bae04950a5d15dc346de in 486ms, sequenceid=289, compaction requested=true 2024-11-26T20:41:49,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:49,255 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:49,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:41:49,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:49,257 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:49,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:41:49,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:49,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:41:49,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:49,259 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:49,259 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/B is initiating minor compaction (all files) 2024-11-26T20:41:49,259 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/B in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:49,259 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/dcdbb8563fcb4077a40e4ff0456f3a6d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/2ed5f5e0b924418e8912793c6918557a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/37b7e851fb9f4a8687d4e4691454487c] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=36.4 K 2024-11-26T20:41:49,259 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:49,259 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/A is initiating minor compaction (all files) 2024-11-26T20:41:49,260 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/A in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:49,260 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/be2a2106329943e186b1a4727c290b9a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/12a334717bf44a679037b32c49c1fabf, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/b95789dee17a45dab717eb0a3b68098f] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=38.8 K 2024-11-26T20:41:49,260 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting dcdbb8563fcb4077a40e4ff0456f3a6d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732653704583 2024-11-26T20:41:49,260 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ed5f5e0b924418e8912793c6918557a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732653706832 2024-11-26T20:41:49,260 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting be2a2106329943e186b1a4727c290b9a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732653704583 2024-11-26T20:41:49,260 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 37b7e851fb9f4a8687d4e4691454487c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732653708756 2024-11-26T20:41:49,261 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12a334717bf44a679037b32c49c1fabf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732653706832 2024-11-26T20:41:49,261 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting b95789dee17a45dab717eb0a3b68098f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732653708756 2024-11-26T20:41:49,270 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#B#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:49,271 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/1deb01b7ec774ebca99d1d5d7913c674 is 50, key is test_row_0/B:col10/1732653708756/Put/seqid=0 2024-11-26T20:41:49,272 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#A#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:49,273 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/b30d8fc7e3a542cb8a225c1788f48a9a is 50, key is test_row_0/A:col10/1732653708756/Put/seqid=0 2024-11-26T20:41:49,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742127_1303 (size=12915) 2024-11-26T20:41:49,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742128_1304 (size=12915) 2024-11-26T20:41:49,305 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/b30d8fc7e3a542cb8a225c1788f48a9a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/b30d8fc7e3a542cb8a225c1788f48a9a 2024-11-26T20:41:49,310 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/A of 38de20de4b25bae04950a5d15dc346de into b30d8fc7e3a542cb8a225c1788f48a9a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:49,310 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:49,310 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/A, priority=13, startTime=1732653709255; duration=0sec 2024-11-26T20:41:49,311 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:49,311 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:A 2024-11-26T20:41:49,311 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:49,312 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:49,312 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/C is initiating minor compaction (all files) 2024-11-26T20:41:49,312 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/C in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:49,312 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/d89b88f20b1e43c7802ca4f81b3c3ccd, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/00f21f1315ab416088ae6e639bda6c3d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/c700e6b4248d49a0b00294236d1a7750] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=36.4 K 2024-11-26T20:41:49,313 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting d89b88f20b1e43c7802ca4f81b3c3ccd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732653704583 2024-11-26T20:41:49,317 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00f21f1315ab416088ae6e639bda6c3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732653706832 2024-11-26T20:41:49,318 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting c700e6b4248d49a0b00294236d1a7750, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732653708756 2024-11-26T20:41:49,325 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#C#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:49,325 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/53b67208a3ec4055aa05c5c12bff7d9d is 50, key is test_row_0/C:col10/1732653708756/Put/seqid=0 2024-11-26T20:41:49,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742129_1305 (size=12915) 2024-11-26T20:41:49,340 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/53b67208a3ec4055aa05c5c12bff7d9d as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/53b67208a3ec4055aa05c5c12bff7d9d 2024-11-26T20:41:49,346 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/C of 38de20de4b25bae04950a5d15dc346de into 53b67208a3ec4055aa05c5c12bff7d9d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:49,346 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:49,346 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/C, priority=13, startTime=1732653709257; duration=0sec 2024-11-26T20:41:49,346 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:49,346 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:C 2024-11-26T20:41:49,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T20:41:49,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-26T20:41:49,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:49,407 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 38de20de4b25bae04950a5d15dc346de 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-26T20:41:49,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=A 2024-11-26T20:41:49,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:49,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=B 2024-11-26T20:41:49,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:49,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=C 2024-11-26T20:41:49,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:41:49,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/58c1789fec0e4cf49d87b47a36933350 is 50, key is test_row_0/A:col10/1732653708910/Put/seqid=0 2024-11-26T20:41:49,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742130_1306 (size=12301) 2024-11-26T20:41:49,427 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/58c1789fec0e4cf49d87b47a36933350 2024-11-26T20:41:49,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/06203ef2e2424e08b26dded68cbf0df0 is 50, key is test_row_0/B:col10/1732653708910/Put/seqid=0 2024-11-26T20:41:49,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742131_1307 (size=12301) 2024-11-26T20:41:49,445 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/06203ef2e2424e08b26dded68cbf0df0 2024-11-26T20:41:49,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/400eb900930448f4a7793d2c9bab6c76 is 50, key is test_row_0/C:col10/1732653708910/Put/seqid=0 2024-11-26T20:41:49,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742132_1308 (size=12301) 2024-11-26T20:41:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:49,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T20:41:49,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653769562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:49,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:49,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653769567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:49,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:49,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653769674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:49,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:49,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653769674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:49,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:49,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:49,693 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/1deb01b7ec774ebca99d1d5d7913c674 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/1deb01b7ec774ebca99d1d5d7913c674 2024-11-26T20:41:49,699 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/B of 38de20de4b25bae04950a5d15dc346de into 1deb01b7ec774ebca99d1d5d7913c674(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:49,699 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:49,699 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/B, priority=13, startTime=1732653709257; duration=0sec 2024-11-26T20:41:49,699 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:49,699 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:B 2024-11-26T20:41:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T20:41:49,880 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/400eb900930448f4a7793d2c9bab6c76 2024-11-26T20:41:49,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/58c1789fec0e4cf49d87b47a36933350 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/58c1789fec0e4cf49d87b47a36933350 2024-11-26T20:41:49,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653769884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:49,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653769885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:49,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:49,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:49,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:49,895 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/58c1789fec0e4cf49d87b47a36933350, entries=150, sequenceid=328, filesize=12.0 K 2024-11-26T20:41:49,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/06203ef2e2424e08b26dded68cbf0df0 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/06203ef2e2424e08b26dded68cbf0df0 2024-11-26T20:41:49,908 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/06203ef2e2424e08b26dded68cbf0df0, entries=150, sequenceid=328, filesize=12.0 K 2024-11-26T20:41:49,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/400eb900930448f4a7793d2c9bab6c76 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/400eb900930448f4a7793d2c9bab6c76 2024-11-26T20:41:49,927 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/400eb900930448f4a7793d2c9bab6c76, entries=150, sequenceid=328, filesize=12.0 K 2024-11-26T20:41:49,928 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 38de20de4b25bae04950a5d15dc346de in 522ms, sequenceid=328, compaction requested=false 2024-11-26T20:41:49,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:49,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:49,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-26T20:41:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-26T20:41:49,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-26T20:41:49,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 682 msec 2024-11-26T20:41:49,936 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 689 msec 2024-11-26T20:41:50,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:50,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38de20de4b25bae04950a5d15dc346de 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T20:41:50,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=A 2024-11-26T20:41:50,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:50,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=B 2024-11-26T20:41:50,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:50,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=C 2024-11-26T20:41:50,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:50,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/46dad3b198c44ee9abcc3cbc276536c4 is 50, key is test_row_0/A:col10/1732653709552/Put/seqid=0 2024-11-26T20:41:50,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742133_1309 (size=12297) 2024-11-26T20:41:50,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:50,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653770254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:50,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:50,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653770254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:50,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:50,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:50,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:50,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:50,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:50,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:50,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:50,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653770364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:50,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:50,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653770364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:50,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T20:41:50,384 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:50,385 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:41:50,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-26T20:41:50,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-26T20:41:50,387 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:41:50,387 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:41:50,388 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:41:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-26T20:41:50,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-26T20:41:50,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:50,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:50,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:50,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:50,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:50,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:50,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:50,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653770574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:50,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:50,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653770574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:50,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/46dad3b198c44ee9abcc3cbc276536c4 2024-11-26T20:41:50,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/e1e4ec62721a46529cbb35546aa3e652 is 50, key is test_row_0/B:col10/1732653709552/Put/seqid=0 2024-11-26T20:41:50,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742134_1310 (size=9857) 2024-11-26T20:41:50,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-26T20:41:50,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:50,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:50,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:50,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:50,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-26T20:41:50,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-26T20:41:50,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:50,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:50,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:50,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:50,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:50,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653770884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:50,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653770884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:50,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:50,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:50,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:50,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:50,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:50,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:50,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-26T20:41:50,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:50,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:50,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:50,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:50,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:51,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-26T20:41:51,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653771024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:51,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653771024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:51,026 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_1' in 38de20de4b25bae04950a5d15dc346de of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:41:51,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:51,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:51,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:51,026 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 38de20de4b25bae04950a5d15dc346de of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:41:51,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:51,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:51,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:51,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/e1e4ec62721a46529cbb35546aa3e652 2024-11-26T20:41:51,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/326badbcf9c14de28fdb4b33b9b3e999 is 50, key is test_row_0/C:col10/1732653709552/Put/seqid=0 2024-11-26T20:41:51,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742135_1311 (size=9857) 2024-11-26T20:41:51,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-26T20:41:51,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:51,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:51,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:51,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:51,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:51,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-26T20:41:51,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:51,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:51,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:51,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:51,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:51,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653771394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:51,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:51,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653771394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:51,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:51,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:51,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:51,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:51,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:51,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/326badbcf9c14de28fdb4b33b9b3e999 2024-11-26T20:41:51,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/46dad3b198c44ee9abcc3cbc276536c4 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/46dad3b198c44ee9abcc3cbc276536c4 2024-11-26T20:41:51,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/46dad3b198c44ee9abcc3cbc276536c4, entries=150, sequenceid=341, filesize=12.0 K 2024-11-26T20:41:51,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/e1e4ec62721a46529cbb35546aa3e652 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/e1e4ec62721a46529cbb35546aa3e652 2024-11-26T20:41:51,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/e1e4ec62721a46529cbb35546aa3e652, entries=100, sequenceid=341, filesize=9.6 K 2024-11-26T20:41:51,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/326badbcf9c14de28fdb4b33b9b3e999 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/326badbcf9c14de28fdb4b33b9b3e999 2024-11-26T20:41:51,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-26T20:41:51,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:51,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:51,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:51,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:51,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:51,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/326badbcf9c14de28fdb4b33b9b3e999, entries=100, sequenceid=341, filesize=9.6 K 2024-11-26T20:41:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:41:51,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 38de20de4b25bae04950a5d15dc346de in 1265ms, sequenceid=341, compaction requested=true 2024-11-26T20:41:51,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:51,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:41:51,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:51,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:41:51,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:51,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:41:51,461 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:51,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:51,461 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:51,462 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37513 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:51,463 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/A is initiating minor compaction (all files) 2024-11-26T20:41:51,463 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/A in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:51,463 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/b30d8fc7e3a542cb8a225c1788f48a9a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/58c1789fec0e4cf49d87b47a36933350, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/46dad3b198c44ee9abcc3cbc276536c4] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=36.6 K 2024-11-26T20:41:51,463 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:51,463 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/B is initiating minor compaction (all files) 2024-11-26T20:41:51,463 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting b30d8fc7e3a542cb8a225c1788f48a9a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732653708756 2024-11-26T20:41:51,463 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/B in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:51,463 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/1deb01b7ec774ebca99d1d5d7913c674, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/06203ef2e2424e08b26dded68cbf0df0, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/e1e4ec62721a46529cbb35546aa3e652] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=34.3 K 2024-11-26T20:41:51,464 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58c1789fec0e4cf49d87b47a36933350, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732653708888 2024-11-26T20:41:51,464 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 1deb01b7ec774ebca99d1d5d7913c674, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732653708756 2024-11-26T20:41:51,464 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46dad3b198c44ee9abcc3cbc276536c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732653709552 2024-11-26T20:41:51,465 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 06203ef2e2424e08b26dded68cbf0df0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732653708888 2024-11-26T20:41:51,465 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting e1e4ec62721a46529cbb35546aa3e652, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732653709552 2024-11-26T20:41:51,472 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#A#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:51,472 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/c49dba4b6dac45369c75a59de7ba22a9 is 50, key is test_row_0/A:col10/1732653709552/Put/seqid=0 2024-11-26T20:41:51,486 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#B#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:51,487 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/3c04ee81a89b486eac3181db95dbb694 is 50, key is test_row_0/B:col10/1732653709552/Put/seqid=0 2024-11-26T20:41:51,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742136_1312 (size=13017) 2024-11-26T20:41:51,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742137_1313 (size=13017) 2024-11-26T20:41:51,496 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/c49dba4b6dac45369c75a59de7ba22a9 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/c49dba4b6dac45369c75a59de7ba22a9 2024-11-26T20:41:51,500 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/3c04ee81a89b486eac3181db95dbb694 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/3c04ee81a89b486eac3181db95dbb694 2024-11-26T20:41:51,502 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/A of 38de20de4b25bae04950a5d15dc346de into c49dba4b6dac45369c75a59de7ba22a9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:51,502 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:51,502 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/A, priority=13, startTime=1732653711461; duration=0sec 2024-11-26T20:41:51,502 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:51,502 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:A 2024-11-26T20:41:51,503 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:51,504 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:51,504 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/C is initiating minor compaction (all files) 2024-11-26T20:41:51,504 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/C in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:51,505 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/53b67208a3ec4055aa05c5c12bff7d9d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/400eb900930448f4a7793d2c9bab6c76, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/326badbcf9c14de28fdb4b33b9b3e999] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=34.3 K 2024-11-26T20:41:51,505 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/B of 38de20de4b25bae04950a5d15dc346de into 3c04ee81a89b486eac3181db95dbb694(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:51,505 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:51,505 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/B, priority=13, startTime=1732653711461; duration=0sec 2024-11-26T20:41:51,505 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:51,505 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:B 2024-11-26T20:41:51,505 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53b67208a3ec4055aa05c5c12bff7d9d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732653708756 2024-11-26T20:41:51,505 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 400eb900930448f4a7793d2c9bab6c76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732653708888 2024-11-26T20:41:51,506 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 326badbcf9c14de28fdb4b33b9b3e999, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732653709552 2024-11-26T20:41:51,514 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#C#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:51,514 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/cf70a28323404f3e99b641c097c0455f is 50, key is test_row_0/C:col10/1732653709552/Put/seqid=0 2024-11-26T20:41:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-26T20:41:51,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742138_1314 (size=13017) 2024-11-26T20:41:51,534 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/cf70a28323404f3e99b641c097c0455f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/cf70a28323404f3e99b641c097c0455f 2024-11-26T20:41:51,541 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/C of 38de20de4b25bae04950a5d15dc346de into cf70a28323404f3e99b641c097c0455f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:51,541 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:51,541 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/C, priority=13, startTime=1732653711461; duration=0sec 2024-11-26T20:41:51,541 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:51,541 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:C 2024-11-26T20:41:51,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-26T20:41:51,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:51,613 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 38de20de4b25bae04950a5d15dc346de 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T20:41:51,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=A 2024-11-26T20:41:51,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:51,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=B 2024-11-26T20:41:51,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:51,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=C 2024-11-26T20:41:51,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:51,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/4ce9b09bf3bb423abb455bb867c86b3f is 50, key is test_row_0/A:col10/1732653710252/Put/seqid=0 2024-11-26T20:41:51,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742139_1315 (size=12301) 2024-11-26T20:41:52,027 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/4ce9b09bf3bb423abb455bb867c86b3f 2024-11-26T20:41:52,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/121a46da0a4e49f88d9a290e68736f6e is 50, key is test_row_0/B:col10/1732653710252/Put/seqid=0 2024-11-26T20:41:52,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742140_1316 (size=12301) 2024-11-26T20:41:52,046 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/121a46da0a4e49f88d9a290e68736f6e 2024-11-26T20:41:52,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/169085a3fa9d4d22a4181415660d501f is 50, key is test_row_0/C:col10/1732653710252/Put/seqid=0 2024-11-26T20:41:52,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742141_1317 (size=12301) 2024-11-26T20:41:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:52,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:52,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653772443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:52,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:52,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653772444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:52,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:52,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:52,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:52,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:52,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:52,462 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/169085a3fa9d4d22a4181415660d501f 2024-11-26T20:41:52,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/4ce9b09bf3bb423abb455bb867c86b3f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/4ce9b09bf3bb423abb455bb867c86b3f 2024-11-26T20:41:52,470 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/4ce9b09bf3bb423abb455bb867c86b3f, entries=150, sequenceid=364, filesize=12.0 K 2024-11-26T20:41:52,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/121a46da0a4e49f88d9a290e68736f6e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/121a46da0a4e49f88d9a290e68736f6e 2024-11-26T20:41:52,476 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/121a46da0a4e49f88d9a290e68736f6e, entries=150, sequenceid=364, filesize=12.0 K 2024-11-26T20:41:52,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/169085a3fa9d4d22a4181415660d501f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/169085a3fa9d4d22a4181415660d501f 2024-11-26T20:41:52,481 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/169085a3fa9d4d22a4181415660d501f, entries=150, sequenceid=364, filesize=12.0 K 2024-11-26T20:41:52,482 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 38de20de4b25bae04950a5d15dc346de in 869ms, sequenceid=364, compaction requested=false 2024-11-26T20:41:52,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:52,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:52,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-26T20:41:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-26T20:41:52,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-26T20:41:52,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0960 sec 2024-11-26T20:41:52,489 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 2.1020 sec 2024-11-26T20:41:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-26T20:41:52,534 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:52,535 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:41:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-26T20:41:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-26T20:41:52,537 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:41:52,537 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:41:52,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:41:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:52,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38de20de4b25bae04950a5d15dc346de 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T20:41:52,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=A 2024-11-26T20:41:52,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:52,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=B 2024-11-26T20:41:52,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:52,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=C 2024-11-26T20:41:52,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:52,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/96bee8c2b83f41fab233cb0b7cac0c09 is 50, key is test_row_0/A:col10/1732653712556/Put/seqid=0 2024-11-26T20:41:52,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742142_1318 (size=14741) 2024-11-26T20:41:52,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/96bee8c2b83f41fab233cb0b7cac0c09 2024-11-26T20:41:52,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/78e8fc6fa7234be2b23bee44d8b41a6d is 50, key is test_row_0/B:col10/1732653712556/Put/seqid=0 2024-11-26T20:41:52,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742143_1319 (size=12301) 2024-11-26T20:41:52,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/78e8fc6fa7234be2b23bee44d8b41a6d 2024-11-26T20:41:52,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/2ed6afab8f1f45378011a526220bc723 is 50, key is test_row_0/C:col10/1732653712556/Put/seqid=0 2024-11-26T20:41:52,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653772608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:52,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:52,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:52,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653772612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:52,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742144_1320 (size=12301) 2024-11-26T20:41:52,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/2ed6afab8f1f45378011a526220bc723 2024-11-26T20:41:52,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/96bee8c2b83f41fab233cb0b7cac0c09 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/96bee8c2b83f41fab233cb0b7cac0c09 2024-11-26T20:41:52,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/96bee8c2b83f41fab233cb0b7cac0c09, entries=200, sequenceid=378, filesize=14.4 K 2024-11-26T20:41:52,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/78e8fc6fa7234be2b23bee44d8b41a6d as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/78e8fc6fa7234be2b23bee44d8b41a6d 2024-11-26T20:41:52,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/78e8fc6fa7234be2b23bee44d8b41a6d, entries=150, sequenceid=378, filesize=12.0 K 2024-11-26T20:41:52,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/2ed6afab8f1f45378011a526220bc723 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/2ed6afab8f1f45378011a526220bc723 2024-11-26T20:41:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-26T20:41:52,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/2ed6afab8f1f45378011a526220bc723, entries=150, sequenceid=378, filesize=12.0 K 2024-11-26T20:41:52,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 38de20de4b25bae04950a5d15dc346de in 90ms, sequenceid=378, compaction requested=true 2024-11-26T20:41:52,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:52,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:41:52,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:52,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:41:52,647 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:52,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:52,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:41:52,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:52,647 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:52,648 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:52,648 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/B is initiating minor compaction (all files) 2024-11-26T20:41:52,648 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/B in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:52,648 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/3c04ee81a89b486eac3181db95dbb694, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/121a46da0a4e49f88d9a290e68736f6e, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/78e8fc6fa7234be2b23bee44d8b41a6d] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=36.7 K 2024-11-26T20:41:52,650 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:52,650 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/A is initiating minor compaction (all files) 2024-11-26T20:41:52,650 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/A in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:52,650 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/c49dba4b6dac45369c75a59de7ba22a9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/4ce9b09bf3bb423abb455bb867c86b3f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/96bee8c2b83f41fab233cb0b7cac0c09] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=39.1 K 2024-11-26T20:41:52,650 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting c49dba4b6dac45369c75a59de7ba22a9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732653708904 2024-11-26T20:41:52,651 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c04ee81a89b486eac3181db95dbb694, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732653708904 2024-11-26T20:41:52,651 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ce9b09bf3bb423abb455bb867c86b3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732653710245 2024-11-26T20:41:52,651 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 121a46da0a4e49f88d9a290e68736f6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732653710245 2024-11-26T20:41:52,651 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96bee8c2b83f41fab233cb0b7cac0c09, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732653712417 2024-11-26T20:41:52,652 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 78e8fc6fa7234be2b23bee44d8b41a6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732653712425 2024-11-26T20:41:52,663 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#B#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:52,664 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/9cf8b334347e4cb2ab5daedb0de0e052 is 50, key is test_row_0/B:col10/1732653712556/Put/seqid=0 2024-11-26T20:41:52,666 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#A#compaction#280 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:52,666 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/07ee442eb4794bc8942eef410ed4439c is 50, key is test_row_0/A:col10/1732653712556/Put/seqid=0 2024-11-26T20:41:52,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742145_1321 (size=13119) 2024-11-26T20:41:52,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742146_1322 (size=13119) 2024-11-26T20:41:52,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-26T20:41:52,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:52,690 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 38de20de4b25bae04950a5d15dc346de 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T20:41:52,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=A 2024-11-26T20:41:52,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:52,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=B 2024-11-26T20:41:52,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:52,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=C 2024-11-26T20:41:52,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:52,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/77297c8ec5f44849a84d8248f1dad62c is 50, key is test_row_0/A:col10/1732653712598/Put/seqid=0 2024-11-26T20:41:52,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742147_1323 (size=12301) 2024-11-26T20:41:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:52,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:52,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653772751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:52,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:52,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:52,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:52,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653772754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:52,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:52,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:52,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-26T20:41:52,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653772864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:52,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:52,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653772864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:52,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:52,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:52,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:52,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:52,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:53,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653773074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:53,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:53,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653773074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:53,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:53,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:53,083 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/9cf8b334347e4cb2ab5daedb0de0e052 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/9cf8b334347e4cb2ab5daedb0de0e052 2024-11-26T20:41:53,088 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/07ee442eb4794bc8942eef410ed4439c as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/07ee442eb4794bc8942eef410ed4439c 2024-11-26T20:41:53,089 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/B of 38de20de4b25bae04950a5d15dc346de into 9cf8b334347e4cb2ab5daedb0de0e052(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:53,089 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:53,089 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/B, priority=13, startTime=1732653712647; duration=0sec 2024-11-26T20:41:53,089 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:53,089 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:B 2024-11-26T20:41:53,090 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:41:53,090 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:41:53,091 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/C is initiating minor compaction (all files) 2024-11-26T20:41:53,091 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/C in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:53,091 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/cf70a28323404f3e99b641c097c0455f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/169085a3fa9d4d22a4181415660d501f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/2ed6afab8f1f45378011a526220bc723] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=36.7 K 2024-11-26T20:41:53,091 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting cf70a28323404f3e99b641c097c0455f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732653708904 2024-11-26T20:41:53,091 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 169085a3fa9d4d22a4181415660d501f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732653710245 2024-11-26T20:41:53,092 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ed6afab8f1f45378011a526220bc723, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732653712425 2024-11-26T20:41:53,093 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/A of 38de20de4b25bae04950a5d15dc346de into 07ee442eb4794bc8942eef410ed4439c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:53,093 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:53,093 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/A, priority=13, startTime=1732653712647; duration=0sec 2024-11-26T20:41:53,093 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:53,093 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:A 2024-11-26T20:41:53,098 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#C#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:53,099 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/be2b14a072af45c091caebec81abe310 is 50, key is test_row_0/C:col10/1732653712556/Put/seqid=0 2024-11-26T20:41:53,101 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/77297c8ec5f44849a84d8248f1dad62c 2024-11-26T20:41:53,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742148_1324 (size=13119) 2024-11-26T20:41:53,108 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/be2b14a072af45c091caebec81abe310 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/be2b14a072af45c091caebec81abe310 2024-11-26T20:41:53,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/3c518b83e3364808a54e6e723cd7120c is 50, key is test_row_0/B:col10/1732653712598/Put/seqid=0 2024-11-26T20:41:53,114 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/C of 38de20de4b25bae04950a5d15dc346de into be2b14a072af45c091caebec81abe310(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:53,114 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:53,114 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/C, priority=13, startTime=1732653712647; duration=0sec 2024-11-26T20:41:53,114 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:53,114 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:C 2024-11-26T20:41:53,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742149_1325 (size=12301) 2024-11-26T20:41:53,118 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/3c518b83e3364808a54e6e723cd7120c 2024-11-26T20:41:53,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/7167d9577b494b2a94e9c1775c12aecb is 50, key is test_row_0/C:col10/1732653712598/Put/seqid=0 2024-11-26T20:41:53,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742150_1326 (size=12301) 2024-11-26T20:41:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-26T20:41:53,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653773383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:53,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:53,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:53,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:53,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:53,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653773384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:53,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:53,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:53,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:53,537 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/7167d9577b494b2a94e9c1775c12aecb 2024-11-26T20:41:53,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/77297c8ec5f44849a84d8248f1dad62c as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/77297c8ec5f44849a84d8248f1dad62c 2024-11-26T20:41:53,545 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/77297c8ec5f44849a84d8248f1dad62c, entries=150, sequenceid=396, filesize=12.0 K 2024-11-26T20:41:53,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/3c518b83e3364808a54e6e723cd7120c as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/3c518b83e3364808a54e6e723cd7120c 2024-11-26T20:41:53,549 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/3c518b83e3364808a54e6e723cd7120c, entries=150, sequenceid=396, filesize=12.0 K 2024-11-26T20:41:53,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/7167d9577b494b2a94e9c1775c12aecb as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/7167d9577b494b2a94e9c1775c12aecb 2024-11-26T20:41:53,554 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/7167d9577b494b2a94e9c1775c12aecb, entries=150, sequenceid=396, filesize=12.0 K 2024-11-26T20:41:53,556 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 38de20de4b25bae04950a5d15dc346de in 866ms, sequenceid=396, compaction requested=false 2024-11-26T20:41:53,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:53,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:53,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-26T20:41:53,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-26T20:41:53,559 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-26T20:41:53,560 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0200 sec 2024-11-26T20:41:53,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.0250 sec 2024-11-26T20:41:53,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-26T20:41:53,674 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:53,674 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:41:53,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-26T20:41:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-26T20:41:53,676 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:41:53,677 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:41:53,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:41:53,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-26T20:41:53,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-26T20:41:53,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:53,829 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 38de20de4b25bae04950a5d15dc346de 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T20:41:53,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=A 2024-11-26T20:41:53,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:53,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=B 2024-11-26T20:41:53,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:53,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=C 2024-11-26T20:41:53,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:53,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/e65464533ed04466a3f469daf278086a is 50, key is test_row_0/A:col10/1732653712752/Put/seqid=0 2024-11-26T20:41:53,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742151_1327 (size=12301) 2024-11-26T20:41:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:53,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. as already flushing 2024-11-26T20:41:53,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653773929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:53,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653773930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:53,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:53,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:53,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-26T20:41:54,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653774034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:54,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:54,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:54,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:54,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653774044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:54,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:54,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:54,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:54,239 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/e65464533ed04466a3f469daf278086a 2024-11-26T20:41:54,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653774244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:54,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:54,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:54,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:54,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/edf5dbaf13f34cb5bba33d9ad21dc5bd is 50, key is test_row_0/B:col10/1732653712752/Put/seqid=0 2024-11-26T20:41:54,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742152_1328 (size=12301) 2024-11-26T20:41:54,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653774254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:54,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:54,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-26T20:41:54,518 INFO [Thread-1164 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1164. 2024-11-26T20:41:54,518 DEBUG [Thread-1164 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:41:54,518 DEBUG [Thread-1164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:54,518 DEBUG [Thread-1164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:54,518 INFO [Thread-1170 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1170. 2024-11-26T20:41:54,518 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:41:54,518 DEBUG [Thread-1170 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:41:54,518 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:54,518 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:41:54,518 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:54,519 INFO [Thread-1166 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1166. 2024-11-26T20:41:54,519 DEBUG [Thread-1166 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:41:54,519 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:54,519 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:54,519 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:41:54,520 INFO [Thread-1168 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1168. 2024-11-26T20:41:54,520 DEBUG [Thread-1168 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:41:54,520 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:54,520 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:54,520 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:41:54,521 INFO [Thread-1162 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1162. 2024-11-26T20:41:54,521 DEBUG [Thread-1162 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:41:54,521 DEBUG [Thread-1162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:54,521 DEBUG [Thread-1162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:54,521 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:41:54,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653774553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:54,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:54,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:54,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:54,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1732653774564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:54,650 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/edf5dbaf13f34cb5bba33d9ad21dc5bd 2024-11-26T20:41:54,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/4713cd7179974e298a4d9901f61f9673 is 50, key is test_row_0/C:col10/1732653712752/Put/seqid=0 2024-11-26T20:41:54,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742153_1329 (size=12301) 2024-11-26T20:41:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-26T20:41:55,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1732653775054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:55,055 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_1' in 38de20de4b25bae04950a5d15dc346de of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:41:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:55,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:55,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:55,060 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/4713cd7179974e298a4d9901f61f9673 2024-11-26T20:41:55,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/e65464533ed04466a3f469daf278086a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e65464533ed04466a3f469daf278086a 2024-11-26T20:41:55,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:55,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:41:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1732653775064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35810 deadline: 1732653775064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 2024-11-26T20:41:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:55,064 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 38de20de4b25bae04950a5d15dc346de of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:41:55,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:55,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:41:55,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:41:55,067 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e65464533ed04466a3f469daf278086a, entries=150, sequenceid=413, filesize=12.0 K 2024-11-26T20:41:55,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/edf5dbaf13f34cb5bba33d9ad21dc5bd as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/edf5dbaf13f34cb5bba33d9ad21dc5bd 2024-11-26T20:41:55,070 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/edf5dbaf13f34cb5bba33d9ad21dc5bd, entries=150, sequenceid=413, filesize=12.0 K 2024-11-26T20:41:55,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/4713cd7179974e298a4d9901f61f9673 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4713cd7179974e298a4d9901f61f9673 2024-11-26T20:41:55,073 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4713cd7179974e298a4d9901f61f9673, entries=150, sequenceid=413, filesize=12.0 K 2024-11-26T20:41:55,074 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 38de20de4b25bae04950a5d15dc346de in 1245ms, sequenceid=413, compaction requested=true 2024-11-26T20:41:55,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:55,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:55,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-26T20:41:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-26T20:41:55,076 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-26T20:41:55,076 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3980 sec 2024-11-26T20:41:55,078 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 1.4030 sec 2024-11-26T20:41:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:41:55,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 38de20de4b25bae04950a5d15dc346de 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T20:41:55,083 INFO [Thread-1157 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1157. 2024-11-26T20:41:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=A 2024-11-26T20:41:55,083 DEBUG [Thread-1157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:41:55,083 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=B 2024-11-26T20:41:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:55,083 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=C 2024-11-26T20:41:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:41:55,084 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:41:55,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/e1785ac83fe546d3a4b0071e00658f5f is 50, key is test_row_0/A:col10/1732653713904/Put/seqid=0 2024-11-26T20:41:55,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742154_1330 (size=12301) 2024-11-26T20:41:55,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/e1785ac83fe546d3a4b0071e00658f5f 2024-11-26T20:41:55,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/838dc42208f3412db2172e450f5da22e is 50, key is test_row_0/B:col10/1732653713904/Put/seqid=0 2024-11-26T20:41:55,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742155_1331 (size=12301) 2024-11-26T20:41:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-26T20:41:55,824 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:41:55,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/838dc42208f3412db2172e450f5da22e 2024-11-26T20:41:55,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/80df196ba74f4e009b5a378042e21e4e is 50, key is test_row_0/C:col10/1732653713904/Put/seqid=0 2024-11-26T20:41:55,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742156_1332 (size=12301) 2024-11-26T20:41:56,085 INFO [Thread-1159 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1159. 2024-11-26T20:41:56,085 DEBUG [Thread-1159 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:41:56,085 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:56,085 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:56,085 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:41:56,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/80df196ba74f4e009b5a378042e21e4e 2024-11-26T20:41:56,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/e1785ac83fe546d3a4b0071e00658f5f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e1785ac83fe546d3a4b0071e00658f5f 2024-11-26T20:41:56,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e1785ac83fe546d3a4b0071e00658f5f, entries=150, sequenceid=433, filesize=12.0 K 2024-11-26T20:41:56,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/838dc42208f3412db2172e450f5da22e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/838dc42208f3412db2172e450f5da22e 2024-11-26T20:41:56,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/838dc42208f3412db2172e450f5da22e, entries=150, sequenceid=433, filesize=12.0 K 2024-11-26T20:41:56,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/80df196ba74f4e009b5a378042e21e4e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/80df196ba74f4e009b5a378042e21e4e 2024-11-26T20:41:56,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/80df196ba74f4e009b5a378042e21e4e, entries=150, sequenceid=433, filesize=12.0 K 2024-11-26T20:41:56,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=6.71 KB/6870 for 38de20de4b25bae04950a5d15dc346de in 1239ms, sequenceid=433, compaction requested=true 2024-11-26T20:41:56,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:56,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:41:56,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:56,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:41:56,323 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:41:56,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:56,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 38de20de4b25bae04950a5d15dc346de:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:41:56,323 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:41:56,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:56,324 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:41:56,324 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:41:56,324 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/A is initiating minor compaction (all files) 2024-11-26T20:41:56,324 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/B is initiating minor compaction (all files) 2024-11-26T20:41:56,324 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/A in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:56,324 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/B in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:56,324 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/07ee442eb4794bc8942eef410ed4439c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/77297c8ec5f44849a84d8248f1dad62c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e65464533ed04466a3f469daf278086a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e1785ac83fe546d3a4b0071e00658f5f] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=48.8 K 2024-11-26T20:41:56,324 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/9cf8b334347e4cb2ab5daedb0de0e052, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/3c518b83e3364808a54e6e723cd7120c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/edf5dbaf13f34cb5bba33d9ad21dc5bd, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/838dc42208f3412db2172e450f5da22e] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=48.8 K 2024-11-26T20:41:56,324 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cf8b334347e4cb2ab5daedb0de0e052, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732653712425 2024-11-26T20:41:56,324 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07ee442eb4794bc8942eef410ed4439c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732653712425 2024-11-26T20:41:56,325 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c518b83e3364808a54e6e723cd7120c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732653712598 2024-11-26T20:41:56,325 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77297c8ec5f44849a84d8248f1dad62c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732653712598 2024-11-26T20:41:56,325 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting edf5dbaf13f34cb5bba33d9ad21dc5bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732653712737 2024-11-26T20:41:56,325 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting e65464533ed04466a3f469daf278086a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732653712737 2024-11-26T20:41:56,325 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 838dc42208f3412db2172e450f5da22e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732653713904 2024-11-26T20:41:56,325 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1785ac83fe546d3a4b0071e00658f5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732653713904 2024-11-26T20:41:56,333 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#A#compaction#291 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:56,333 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#B#compaction#292 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:56,333 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/e2e5d1a237e54320a351771cc23d4cd7 is 50, key is test_row_0/A:col10/1732653713904/Put/seqid=0 2024-11-26T20:41:56,333 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/765aaecaf41b40caac7c7ce7d4d1f750 is 50, key is test_row_0/B:col10/1732653713904/Put/seqid=0 2024-11-26T20:41:56,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742158_1334 (size=13255) 2024-11-26T20:41:56,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742157_1333 (size=13255) 2024-11-26T20:41:56,741 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/e2e5d1a237e54320a351771cc23d4cd7 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e2e5d1a237e54320a351771cc23d4cd7 2024-11-26T20:41:56,741 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/765aaecaf41b40caac7c7ce7d4d1f750 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/765aaecaf41b40caac7c7ce7d4d1f750 2024-11-26T20:41:56,745 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/B of 38de20de4b25bae04950a5d15dc346de into 765aaecaf41b40caac7c7ce7d4d1f750(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:56,745 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/A of 38de20de4b25bae04950a5d15dc346de into e2e5d1a237e54320a351771cc23d4cd7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:56,745 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:56,745 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:56,745 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/B, priority=12, startTime=1732653716323; duration=0sec 2024-11-26T20:41:56,745 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/A, priority=12, startTime=1732653716323; duration=0sec 2024-11-26T20:41:56,745 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:56,745 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:B 2024-11-26T20:41:56,745 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:41:56,745 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:A 2024-11-26T20:41:56,745 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:41:56,746 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:41:56,746 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 38de20de4b25bae04950a5d15dc346de/C is initiating minor compaction (all files) 2024-11-26T20:41:56,746 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 38de20de4b25bae04950a5d15dc346de/C in TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:41:56,746 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/be2b14a072af45c091caebec81abe310, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/7167d9577b494b2a94e9c1775c12aecb, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4713cd7179974e298a4d9901f61f9673, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/80df196ba74f4e009b5a378042e21e4e] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp, totalSize=48.8 K 2024-11-26T20:41:56,746 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting be2b14a072af45c091caebec81abe310, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732653712425 2024-11-26T20:41:56,747 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 7167d9577b494b2a94e9c1775c12aecb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732653712598 2024-11-26T20:41:56,747 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 4713cd7179974e298a4d9901f61f9673, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732653712737 2024-11-26T20:41:56,747 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 80df196ba74f4e009b5a378042e21e4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732653713904 2024-11-26T20:41:56,754 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 38de20de4b25bae04950a5d15dc346de#C#compaction#293 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:41:56,754 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/a327435f10374019a5d7ce4f5819fbe1 is 50, key is test_row_0/C:col10/1732653713904/Put/seqid=0 2024-11-26T20:41:56,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742159_1335 (size=13255) 2024-11-26T20:41:57,161 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/a327435f10374019a5d7ce4f5819fbe1 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/a327435f10374019a5d7ce4f5819fbe1 2024-11-26T20:41:57,165 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 38de20de4b25bae04950a5d15dc346de/C of 38de20de4b25bae04950a5d15dc346de into a327435f10374019a5d7ce4f5819fbe1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:41:57,165 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 38de20de4b25bae04950a5d15dc346de: 2024-11-26T20:41:57,165 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., storeName=38de20de4b25bae04950a5d15dc346de/C, priority=12, startTime=1732653716323; duration=0sec 2024-11-26T20:41:57,165 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:41:57,165 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 38de20de4b25bae04950a5d15dc346de:C 2024-11-26T20:41:57,725 INFO [Thread-1153 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1153. 2024-11-26T20:41:57,725 DEBUG [Thread-1153 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:41:57,725 DEBUG [Thread-1153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:57,725 DEBUG [Thread-1153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:41:57,725 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:41:59,966 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T20:42:05,075 INFO [Thread-1151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1151. 2024-11-26T20:42:05,075 DEBUG [Thread-1151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:42:05,075 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:05,075 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:05,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:42:05,105 INFO [Thread-1155 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1155. 2024-11-26T20:42:05,105 DEBUG [Thread-1155 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:42:05,106 DEBUG [Thread-1155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:05,106 DEBUG [Thread-1155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:05,106 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:42:05,106 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T20:42:05,106 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-26T20:42:05,106 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-26T20:42:05,106 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-26T20:42:05,106 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-26T20:42:05,106 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-26T20:42:05,106 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T20:42:05,106 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2933 2024-11-26T20:42:05,106 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2952 2024-11-26T20:42:05,106 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3112 2024-11-26T20:42:05,106 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2927 2024-11-26T20:42:05,106 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2948 2024-11-26T20:42:05,106 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T20:42:05,106 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T20:42:05,106 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:42:05,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:05,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:05,107 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:42:05,107 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T20:42:05,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T20:42:05,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-26T20:42:05,111 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653725111"}]},"ts":"1732653725111"} 2024-11-26T20:42:05,112 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T20:42:05,112 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T20:42:05,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T20:42:05,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38de20de4b25bae04950a5d15dc346de, UNASSIGN}] 2024-11-26T20:42:05,115 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38de20de4b25bae04950a5d15dc346de, UNASSIGN 2024-11-26T20:42:05,116 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=38de20de4b25bae04950a5d15dc346de, regionState=CLOSING, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:42:05,117 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=38de20de4b25bae04950a5d15dc346de, UNASSIGN because future has completed 2024-11-26T20:42:05,117 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T20:42:05,117 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795}] 2024-11-26T20:42:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-26T20:42:05,269 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:42:05,269 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T20:42:05,269 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 38de20de4b25bae04950a5d15dc346de, disabling compactions & flushes 2024-11-26T20:42:05,269 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:42:05,269 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:42:05,269 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. after waiting 0 ms 2024-11-26T20:42:05,269 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:42:05,269 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(2902): Flushing 38de20de4b25bae04950a5d15dc346de 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-26T20:42:05,269 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=A 2024-11-26T20:42:05,269 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:05,269 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=B 2024-11-26T20:42:05,269 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:05,269 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 38de20de4b25bae04950a5d15dc346de, store=C 2024-11-26T20:42:05,269 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:05,273 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/e49c0621087646be9a9ad27a52b7f11e is 50, key is test_row_0/A:col10/1732653725074/Put/seqid=0 2024-11-26T20:42:05,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742160_1336 (size=12301) 2024-11-26T20:42:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-26T20:42:05,677 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/e49c0621087646be9a9ad27a52b7f11e 2024-11-26T20:42:05,684 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/46d804f6db74450f9d940627c6acb437 is 50, key is test_row_0/B:col10/1732653725074/Put/seqid=0 2024-11-26T20:42:05,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742161_1337 (size=12301) 2024-11-26T20:42:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-26T20:42:06,088 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/46d804f6db74450f9d940627c6acb437 2024-11-26T20:42:06,094 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/119992285c234e6ca00965a0242a7140 is 50, key is test_row_0/C:col10/1732653725074/Put/seqid=0 2024-11-26T20:42:06,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742162_1338 (size=12301) 2024-11-26T20:42:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-26T20:42:06,497 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/119992285c234e6ca00965a0242a7140 2024-11-26T20:42:06,501 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/A/e49c0621087646be9a9ad27a52b7f11e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e49c0621087646be9a9ad27a52b7f11e 2024-11-26T20:42:06,504 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e49c0621087646be9a9ad27a52b7f11e, entries=150, sequenceid=443, filesize=12.0 K 2024-11-26T20:42:06,505 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/B/46d804f6db74450f9d940627c6acb437 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/46d804f6db74450f9d940627c6acb437 2024-11-26T20:42:06,508 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/46d804f6db74450f9d940627c6acb437, entries=150, sequenceid=443, filesize=12.0 K 2024-11-26T20:42:06,508 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/.tmp/C/119992285c234e6ca00965a0242a7140 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/119992285c234e6ca00965a0242a7140 2024-11-26T20:42:06,512 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/119992285c234e6ca00965a0242a7140, entries=150, sequenceid=443, filesize=12.0 K 2024-11-26T20:42:06,512 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 38de20de4b25bae04950a5d15dc346de in 1243ms, sequenceid=443, compaction requested=false 2024-11-26T20:42:06,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/96a8626f51de474890b5df0987c18261, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/db24027e0558484881dc10249d5e5474, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/ea4e0750515d4cedbfed0ba0730e1fc8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/863039a46b584d69bdad265b20225dfa, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/f5987d5dc6ac4b369937dc185fa3e5bf, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/06b0f359ab574ccea96a191af14448cd, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/764e5666a89840beb1cbe77b784aa85c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/caec1d368b0942c281661589d882bb2b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/18af93be3a9b4c2f96e94a48879eb606, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/1c1479398fab4aa7ae02f5bd47c866e4, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e2ad7981d9224a1898b7322c0aaac5cf, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/31cad0aa5ceb45919d24d464ef67419a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/c13d7586681a4847ba2dd01bf8d897ab, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/56eb7709a5bd4b5ea55a019c75a8895a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/be2a2106329943e186b1a4727c290b9a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/094fb5ae219b4bd5af4ff07fcd6f2a3b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/12a334717bf44a679037b32c49c1fabf, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/b95789dee17a45dab717eb0a3b68098f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/b30d8fc7e3a542cb8a225c1788f48a9a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/58c1789fec0e4cf49d87b47a36933350, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/c49dba4b6dac45369c75a59de7ba22a9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/46dad3b198c44ee9abcc3cbc276536c4, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/4ce9b09bf3bb423abb455bb867c86b3f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/96bee8c2b83f41fab233cb0b7cac0c09, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/07ee442eb4794bc8942eef410ed4439c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/77297c8ec5f44849a84d8248f1dad62c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e65464533ed04466a3f469daf278086a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e1785ac83fe546d3a4b0071e00658f5f] to archive 2024-11-26T20:42:06,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T20:42:06,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/96a8626f51de474890b5df0987c18261 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/96a8626f51de474890b5df0987c18261 2024-11-26T20:42:06,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/db24027e0558484881dc10249d5e5474 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/db24027e0558484881dc10249d5e5474 2024-11-26T20:42:06,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/ea4e0750515d4cedbfed0ba0730e1fc8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/ea4e0750515d4cedbfed0ba0730e1fc8 2024-11-26T20:42:06,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/863039a46b584d69bdad265b20225dfa to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/863039a46b584d69bdad265b20225dfa 2024-11-26T20:42:06,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/f5987d5dc6ac4b369937dc185fa3e5bf to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/f5987d5dc6ac4b369937dc185fa3e5bf 2024-11-26T20:42:06,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/06b0f359ab574ccea96a191af14448cd to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/06b0f359ab574ccea96a191af14448cd 2024-11-26T20:42:06,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/764e5666a89840beb1cbe77b784aa85c to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/764e5666a89840beb1cbe77b784aa85c 2024-11-26T20:42:06,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/caec1d368b0942c281661589d882bb2b to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/caec1d368b0942c281661589d882bb2b 2024-11-26T20:42:06,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/18af93be3a9b4c2f96e94a48879eb606 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/18af93be3a9b4c2f96e94a48879eb606 2024-11-26T20:42:06,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/1c1479398fab4aa7ae02f5bd47c866e4 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/1c1479398fab4aa7ae02f5bd47c866e4 2024-11-26T20:42:06,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e2ad7981d9224a1898b7322c0aaac5cf to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e2ad7981d9224a1898b7322c0aaac5cf 2024-11-26T20:42:06,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/31cad0aa5ceb45919d24d464ef67419a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/31cad0aa5ceb45919d24d464ef67419a 2024-11-26T20:42:06,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/c13d7586681a4847ba2dd01bf8d897ab to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/c13d7586681a4847ba2dd01bf8d897ab 2024-11-26T20:42:06,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/56eb7709a5bd4b5ea55a019c75a8895a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/56eb7709a5bd4b5ea55a019c75a8895a 2024-11-26T20:42:06,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/be2a2106329943e186b1a4727c290b9a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/be2a2106329943e186b1a4727c290b9a 2024-11-26T20:42:06,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/094fb5ae219b4bd5af4ff07fcd6f2a3b to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/094fb5ae219b4bd5af4ff07fcd6f2a3b 2024-11-26T20:42:06,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/12a334717bf44a679037b32c49c1fabf to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/12a334717bf44a679037b32c49c1fabf 2024-11-26T20:42:06,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/b95789dee17a45dab717eb0a3b68098f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/b95789dee17a45dab717eb0a3b68098f 2024-11-26T20:42:06,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/b30d8fc7e3a542cb8a225c1788f48a9a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/b30d8fc7e3a542cb8a225c1788f48a9a 2024-11-26T20:42:06,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/58c1789fec0e4cf49d87b47a36933350 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/58c1789fec0e4cf49d87b47a36933350 2024-11-26T20:42:06,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/c49dba4b6dac45369c75a59de7ba22a9 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/c49dba4b6dac45369c75a59de7ba22a9 2024-11-26T20:42:06,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/46dad3b198c44ee9abcc3cbc276536c4 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/46dad3b198c44ee9abcc3cbc276536c4 2024-11-26T20:42:06,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/4ce9b09bf3bb423abb455bb867c86b3f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/4ce9b09bf3bb423abb455bb867c86b3f 2024-11-26T20:42:06,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/96bee8c2b83f41fab233cb0b7cac0c09 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/96bee8c2b83f41fab233cb0b7cac0c09 2024-11-26T20:42:06,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/07ee442eb4794bc8942eef410ed4439c to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/07ee442eb4794bc8942eef410ed4439c 2024-11-26T20:42:06,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/77297c8ec5f44849a84d8248f1dad62c to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/77297c8ec5f44849a84d8248f1dad62c 2024-11-26T20:42:06,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e65464533ed04466a3f469daf278086a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e65464533ed04466a3f469daf278086a 2024-11-26T20:42:06,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e1785ac83fe546d3a4b0071e00658f5f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e1785ac83fe546d3a4b0071e00658f5f 2024-11-26T20:42:06,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/c6b48a1180c24e3db785c7b6cb17eb1d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/14783aea350e489cb8e0d1b36cbb5892, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/eec6d554c0d1419a804fc66aeb57554c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/ba9022b90e83479db422c950cfb600bb, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/dfe06fa8f40241d692203333456474fd, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/2895cfcd9e21489d9cdfde9843e2d885, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/71c7e7da2e01421cbaeb2d9b4da1f643, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/fcd4eddfdb094f15b02013c9253ccb21, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/0cd470c382e74b84a80b2f97d21ae197, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/733aba2add344a4cb7757faf8caf5e87, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/73265d5c2a4841efb78a9dca1479f831, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/02137ae1bfb340709d97a013df9bfb7a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/9b74add8d2494e33872a17d215d03eb2, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/13d26e25174647fcb159c7e078c67ba1, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/dcdbb8563fcb4077a40e4ff0456f3a6d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/a5624676890c44f7af408ed5b131ae0f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/2ed5f5e0b924418e8912793c6918557a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/1deb01b7ec774ebca99d1d5d7913c674, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/37b7e851fb9f4a8687d4e4691454487c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/06203ef2e2424e08b26dded68cbf0df0, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/3c04ee81a89b486eac3181db95dbb694, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/e1e4ec62721a46529cbb35546aa3e652, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/121a46da0a4e49f88d9a290e68736f6e, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/9cf8b334347e4cb2ab5daedb0de0e052, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/78e8fc6fa7234be2b23bee44d8b41a6d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/3c518b83e3364808a54e6e723cd7120c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/edf5dbaf13f34cb5bba33d9ad21dc5bd, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/838dc42208f3412db2172e450f5da22e] to archive 2024-11-26T20:42:06,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T20:42:06,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/c6b48a1180c24e3db785c7b6cb17eb1d to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/c6b48a1180c24e3db785c7b6cb17eb1d 2024-11-26T20:42:06,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/14783aea350e489cb8e0d1b36cbb5892 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/14783aea350e489cb8e0d1b36cbb5892 2024-11-26T20:42:06,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/eec6d554c0d1419a804fc66aeb57554c to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/eec6d554c0d1419a804fc66aeb57554c 2024-11-26T20:42:06,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/ba9022b90e83479db422c950cfb600bb to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/ba9022b90e83479db422c950cfb600bb 2024-11-26T20:42:06,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/dfe06fa8f40241d692203333456474fd to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/dfe06fa8f40241d692203333456474fd 2024-11-26T20:42:06,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/2895cfcd9e21489d9cdfde9843e2d885 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/2895cfcd9e21489d9cdfde9843e2d885 2024-11-26T20:42:06,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/71c7e7da2e01421cbaeb2d9b4da1f643 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/71c7e7da2e01421cbaeb2d9b4da1f643 2024-11-26T20:42:06,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/fcd4eddfdb094f15b02013c9253ccb21 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/fcd4eddfdb094f15b02013c9253ccb21 2024-11-26T20:42:06,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/0cd470c382e74b84a80b2f97d21ae197 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/0cd470c382e74b84a80b2f97d21ae197 2024-11-26T20:42:06,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/733aba2add344a4cb7757faf8caf5e87 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/733aba2add344a4cb7757faf8caf5e87 2024-11-26T20:42:06,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/73265d5c2a4841efb78a9dca1479f831 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/73265d5c2a4841efb78a9dca1479f831 2024-11-26T20:42:06,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/02137ae1bfb340709d97a013df9bfb7a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/02137ae1bfb340709d97a013df9bfb7a 2024-11-26T20:42:06,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/9b74add8d2494e33872a17d215d03eb2 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/9b74add8d2494e33872a17d215d03eb2 2024-11-26T20:42:06,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/13d26e25174647fcb159c7e078c67ba1 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/13d26e25174647fcb159c7e078c67ba1 2024-11-26T20:42:06,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/dcdbb8563fcb4077a40e4ff0456f3a6d to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/dcdbb8563fcb4077a40e4ff0456f3a6d 2024-11-26T20:42:06,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/a5624676890c44f7af408ed5b131ae0f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/a5624676890c44f7af408ed5b131ae0f 2024-11-26T20:42:06,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/2ed5f5e0b924418e8912793c6918557a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/2ed5f5e0b924418e8912793c6918557a 2024-11-26T20:42:06,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/1deb01b7ec774ebca99d1d5d7913c674 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/1deb01b7ec774ebca99d1d5d7913c674 2024-11-26T20:42:06,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/37b7e851fb9f4a8687d4e4691454487c to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/37b7e851fb9f4a8687d4e4691454487c 2024-11-26T20:42:06,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/06203ef2e2424e08b26dded68cbf0df0 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/06203ef2e2424e08b26dded68cbf0df0 2024-11-26T20:42:06,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/3c04ee81a89b486eac3181db95dbb694 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/3c04ee81a89b486eac3181db95dbb694 2024-11-26T20:42:06,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/e1e4ec62721a46529cbb35546aa3e652 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/e1e4ec62721a46529cbb35546aa3e652 2024-11-26T20:42:06,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/121a46da0a4e49f88d9a290e68736f6e to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/121a46da0a4e49f88d9a290e68736f6e 2024-11-26T20:42:06,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/9cf8b334347e4cb2ab5daedb0de0e052 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/9cf8b334347e4cb2ab5daedb0de0e052 2024-11-26T20:42:06,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/78e8fc6fa7234be2b23bee44d8b41a6d to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/78e8fc6fa7234be2b23bee44d8b41a6d 2024-11-26T20:42:06,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/3c518b83e3364808a54e6e723cd7120c to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/3c518b83e3364808a54e6e723cd7120c 2024-11-26T20:42:06,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/edf5dbaf13f34cb5bba33d9ad21dc5bd to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/edf5dbaf13f34cb5bba33d9ad21dc5bd 2024-11-26T20:42:06,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/838dc42208f3412db2172e450f5da22e to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/838dc42208f3412db2172e450f5da22e 2024-11-26T20:42:06,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/8db2a83c7b1c4200b4bc54c2be951818, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/97eeb472ea7c46a0aee8adfe881206c2, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/bb5006d640464ecd84ec31233f0b66ce, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/327ce2783900477f8f1ebea081b34111, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/025dea5e84994973b487e38f3772d0eb, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4ccb0b4f8c8041588856622eb4d93cc2, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/2da10ad447dc44b4aa4cf256e9f413be, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/8822bcf0802b4686b92eb46e61fccc59, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/05cc7ce1a4b943319c18bd9832334fe2, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/a40ed1fba02242f5ad0fd903ed1bea02, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/469586fd06bc4a15ba2e1168d504d902, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4a159b94b29d4711a0b9ba3b6bf97678, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/98e055d6b38941f4b02544c9aab7e26d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4446765b0e7c441ca03e5d84efe4204e, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/d89b88f20b1e43c7802ca4f81b3c3ccd, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/6206143a93954b90aaa935d19880d2ca, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/00f21f1315ab416088ae6e639bda6c3d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/53b67208a3ec4055aa05c5c12bff7d9d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/c700e6b4248d49a0b00294236d1a7750, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/400eb900930448f4a7793d2c9bab6c76, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/cf70a28323404f3e99b641c097c0455f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/326badbcf9c14de28fdb4b33b9b3e999, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/169085a3fa9d4d22a4181415660d501f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/be2b14a072af45c091caebec81abe310, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/2ed6afab8f1f45378011a526220bc723, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/7167d9577b494b2a94e9c1775c12aecb, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4713cd7179974e298a4d9901f61f9673, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/80df196ba74f4e009b5a378042e21e4e] to archive 2024-11-26T20:42:06,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T20:42:06,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/8db2a83c7b1c4200b4bc54c2be951818 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/8db2a83c7b1c4200b4bc54c2be951818 2024-11-26T20:42:06,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/97eeb472ea7c46a0aee8adfe881206c2 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/97eeb472ea7c46a0aee8adfe881206c2 2024-11-26T20:42:06,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/bb5006d640464ecd84ec31233f0b66ce to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/bb5006d640464ecd84ec31233f0b66ce 2024-11-26T20:42:06,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/327ce2783900477f8f1ebea081b34111 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/327ce2783900477f8f1ebea081b34111 2024-11-26T20:42:06,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/025dea5e84994973b487e38f3772d0eb to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/025dea5e84994973b487e38f3772d0eb 2024-11-26T20:42:06,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4ccb0b4f8c8041588856622eb4d93cc2 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4ccb0b4f8c8041588856622eb4d93cc2 2024-11-26T20:42:06,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/2da10ad447dc44b4aa4cf256e9f413be to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/2da10ad447dc44b4aa4cf256e9f413be 2024-11-26T20:42:06,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/8822bcf0802b4686b92eb46e61fccc59 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/8822bcf0802b4686b92eb46e61fccc59 2024-11-26T20:42:06,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/05cc7ce1a4b943319c18bd9832334fe2 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/05cc7ce1a4b943319c18bd9832334fe2 2024-11-26T20:42:06,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/a40ed1fba02242f5ad0fd903ed1bea02 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/a40ed1fba02242f5ad0fd903ed1bea02 2024-11-26T20:42:06,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/469586fd06bc4a15ba2e1168d504d902 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/469586fd06bc4a15ba2e1168d504d902 2024-11-26T20:42:06,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4a159b94b29d4711a0b9ba3b6bf97678 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4a159b94b29d4711a0b9ba3b6bf97678 2024-11-26T20:42:06,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/98e055d6b38941f4b02544c9aab7e26d to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/98e055d6b38941f4b02544c9aab7e26d 2024-11-26T20:42:06,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4446765b0e7c441ca03e5d84efe4204e to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4446765b0e7c441ca03e5d84efe4204e 2024-11-26T20:42:06,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/d89b88f20b1e43c7802ca4f81b3c3ccd to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/d89b88f20b1e43c7802ca4f81b3c3ccd 2024-11-26T20:42:06,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/6206143a93954b90aaa935d19880d2ca to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/6206143a93954b90aaa935d19880d2ca 2024-11-26T20:42:06,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/00f21f1315ab416088ae6e639bda6c3d to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/00f21f1315ab416088ae6e639bda6c3d 2024-11-26T20:42:06,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/53b67208a3ec4055aa05c5c12bff7d9d to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/53b67208a3ec4055aa05c5c12bff7d9d 2024-11-26T20:42:06,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/c700e6b4248d49a0b00294236d1a7750 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/c700e6b4248d49a0b00294236d1a7750 2024-11-26T20:42:06,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/400eb900930448f4a7793d2c9bab6c76 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/400eb900930448f4a7793d2c9bab6c76 2024-11-26T20:42:06,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/cf70a28323404f3e99b641c097c0455f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/cf70a28323404f3e99b641c097c0455f 2024-11-26T20:42:06,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/326badbcf9c14de28fdb4b33b9b3e999 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/326badbcf9c14de28fdb4b33b9b3e999 2024-11-26T20:42:06,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/169085a3fa9d4d22a4181415660d501f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/169085a3fa9d4d22a4181415660d501f 2024-11-26T20:42:06,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/be2b14a072af45c091caebec81abe310 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/be2b14a072af45c091caebec81abe310 2024-11-26T20:42:06,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/2ed6afab8f1f45378011a526220bc723 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/2ed6afab8f1f45378011a526220bc723 2024-11-26T20:42:06,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/7167d9577b494b2a94e9c1775c12aecb to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/7167d9577b494b2a94e9c1775c12aecb 2024-11-26T20:42:06,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4713cd7179974e298a4d9901f61f9673 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/4713cd7179974e298a4d9901f61f9673 2024-11-26T20:42:06,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/80df196ba74f4e009b5a378042e21e4e to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/80df196ba74f4e009b5a378042e21e4e 2024-11-26T20:42:06,588 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/recovered.edits/446.seqid, newMaxSeqId=446, maxSeqId=1 2024-11-26T20:42:06,588 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. 2024-11-26T20:42:06,588 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 38de20de4b25bae04950a5d15dc346de: Waiting for close lock at 1732653725269Running coprocessor pre-close hooks at 1732653725269Disabling compacts and flushes for region at 1732653725269Disabling writes for close at 1732653725269Obtaining lock to block concurrent updates at 1732653725269Preparing flush snapshotting stores in 38de20de4b25bae04950a5d15dc346de at 1732653725269Finished memstore snapshotting TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732653725269Flushing stores of TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de. at 1732653725270 (+1 ms)Flushing 38de20de4b25bae04950a5d15dc346de/A: creating writer at 1732653725270Flushing 38de20de4b25bae04950a5d15dc346de/A: appending metadata at 1732653725273 (+3 ms)Flushing 38de20de4b25bae04950a5d15dc346de/A: closing flushed file at 1732653725273Flushing 38de20de4b25bae04950a5d15dc346de/B: creating writer at 1732653725681 (+408 ms)Flushing 38de20de4b25bae04950a5d15dc346de/B: appending metadata at 1732653725684 (+3 ms)Flushing 38de20de4b25bae04950a5d15dc346de/B: closing flushed file at 1732653725684Flushing 38de20de4b25bae04950a5d15dc346de/C: creating writer at 1732653726091 (+407 ms)Flushing 38de20de4b25bae04950a5d15dc346de/C: appending metadata at 1732653726093 (+2 ms)Flushing 38de20de4b25bae04950a5d15dc346de/C: closing flushed file at 1732653726093Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54156672: reopening flushed file at 1732653726501 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6416ca6d: reopening flushed file at 1732653726504 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bb444ea: reopening flushed file at 1732653726508 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 38de20de4b25bae04950a5d15dc346de in 1243ms, sequenceid=443, compaction requested=false at 1732653726512 (+4 ms)Writing region close event to WAL at 1732653726585 (+73 ms)Running coprocessor post-close hooks at 1732653726588 (+3 ms)Closed at 1732653726588 2024-11-26T20:42:06,590 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 38de20de4b25bae04950a5d15dc346de 2024-11-26T20:42:06,591 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=38de20de4b25bae04950a5d15dc346de, regionState=CLOSED 2024-11-26T20:42:06,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 because future has completed 2024-11-26T20:42:06,594 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-26T20:42:06,594 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 38de20de4b25bae04950a5d15dc346de, server=cd6be161543e,40291,1732653631795 in 1.4760 sec 2024-11-26T20:42:06,595 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-26T20:42:06,595 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=38de20de4b25bae04950a5d15dc346de, UNASSIGN in 1.4800 sec 2024-11-26T20:42:06,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-26T20:42:06,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4830 sec 2024-11-26T20:42:06,598 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653726598"}]},"ts":"1732653726598"} 2024-11-26T20:42:06,599 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T20:42:06,599 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T20:42:06,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4920 sec 2024-11-26T20:42:07,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-26T20:42:07,254 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:42:07,254 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T20:42:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:42:07,256 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:42:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-26T20:42:07,256 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=92, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:42:07,258 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de 2024-11-26T20:42:07,260 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A, FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B, FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C, FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/recovered.edits] 2024-11-26T20:42:07,262 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e2e5d1a237e54320a351771cc23d4cd7 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e2e5d1a237e54320a351771cc23d4cd7 2024-11-26T20:42:07,263 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e49c0621087646be9a9ad27a52b7f11e to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/A/e49c0621087646be9a9ad27a52b7f11e 2024-11-26T20:42:07,265 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/46d804f6db74450f9d940627c6acb437 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/46d804f6db74450f9d940627c6acb437 2024-11-26T20:42:07,265 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/765aaecaf41b40caac7c7ce7d4d1f750 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/B/765aaecaf41b40caac7c7ce7d4d1f750 2024-11-26T20:42:07,267 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/119992285c234e6ca00965a0242a7140 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/119992285c234e6ca00965a0242a7140 2024-11-26T20:42:07,268 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/a327435f10374019a5d7ce4f5819fbe1 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/C/a327435f10374019a5d7ce4f5819fbe1 2024-11-26T20:42:07,270 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/recovered.edits/446.seqid to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de/recovered.edits/446.seqid 2024-11-26T20:42:07,270 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/38de20de4b25bae04950a5d15dc346de 2024-11-26T20:42:07,270 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T20:42:07,272 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=92, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:42:07,275 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T20:42:07,277 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T20:42:07,278 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=92, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:42:07,278 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T20:42:07,278 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732653727278"}]},"ts":"9223372036854775807"} 2024-11-26T20:42:07,280 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T20:42:07,280 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 38de20de4b25bae04950a5d15dc346de, NAME => 'TestAcidGuarantees,,1732653692267.38de20de4b25bae04950a5d15dc346de.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T20:42:07,280 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T20:42:07,280 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732653727280"}]},"ts":"9223372036854775807"} 2024-11-26T20:42:07,281 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T20:42:07,282 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=92, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:42:07,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 28 msec 2024-11-26T20:42:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-26T20:42:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T20:42:07,364 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:42:07,372 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 252), OpenFileDescriptor=449 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=402 (was 514), ProcessCount=11 (was 11), AvailableMemoryMB=5763 (was 6401) 2024-11-26T20:42:07,380 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=402, ProcessCount=11, AvailableMemoryMB=5762 2024-11-26T20:42:07,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T20:42:07,382 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T20:42:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T20:42:07,383 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T20:42:07,383 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:07,383 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 93 2024-11-26T20:42:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T20:42:07,384 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T20:42:07,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742163_1339 (size=1061) 2024-11-26T20:42:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T20:42:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T20:42:07,791 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794 2024-11-26T20:42:07,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742164_1340 (size=53) 2024-11-26T20:42:08,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T20:42:08,196 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:42:08,196 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1591a955d2fc691174b77e9b19a3d332, disabling compactions & flushes 2024-11-26T20:42:08,196 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:08,196 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:08,196 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. after waiting 0 ms 2024-11-26T20:42:08,196 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:08,196 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:08,196 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1591a955d2fc691174b77e9b19a3d332: Waiting for close lock at 1732653728196Disabling compacts and flushes for region at 1732653728196Disabling writes for close at 1732653728196Writing region close event to WAL at 1732653728196Closed at 1732653728196 2024-11-26T20:42:08,197 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T20:42:08,198 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732653728197"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732653728197"}]},"ts":"1732653728197"} 2024-11-26T20:42:08,199 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T20:42:08,200 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T20:42:08,200 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653728200"}]},"ts":"1732653728200"} 2024-11-26T20:42:08,201 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T20:42:08,202 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1591a955d2fc691174b77e9b19a3d332, ASSIGN}] 2024-11-26T20:42:08,202 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1591a955d2fc691174b77e9b19a3d332, ASSIGN 2024-11-26T20:42:08,203 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1591a955d2fc691174b77e9b19a3d332, ASSIGN; state=OFFLINE, location=cd6be161543e,40291,1732653631795; forceNewPlan=false, retain=false 2024-11-26T20:42:08,354 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=1591a955d2fc691174b77e9b19a3d332, regionState=OPENING, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:42:08,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1591a955d2fc691174b77e9b19a3d332, ASSIGN because future has completed 2024-11-26T20:42:08,357 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795}] 2024-11-26T20:42:08,512 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:08,513 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7752): Opening region: {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:42:08,513 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:08,513 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:42:08,513 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7794): checking encryption for 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:08,513 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7797): checking classloading for 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:08,515 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:08,516 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:42:08,516 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1591a955d2fc691174b77e9b19a3d332 columnFamilyName A 2024-11-26T20:42:08,516 DEBUG [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:08,517 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] regionserver.HStore(327): Store=1591a955d2fc691174b77e9b19a3d332/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:42:08,517 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:08,518 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:42:08,518 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1591a955d2fc691174b77e9b19a3d332 columnFamilyName B 2024-11-26T20:42:08,519 DEBUG [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:08,519 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] regionserver.HStore(327): Store=1591a955d2fc691174b77e9b19a3d332/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:42:08,519 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:08,520 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:42:08,520 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1591a955d2fc691174b77e9b19a3d332 columnFamilyName C 2024-11-26T20:42:08,520 DEBUG [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:08,521 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] regionserver.HStore(327): Store=1591a955d2fc691174b77e9b19a3d332/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:42:08,521 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:08,521 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1038): replaying wal for 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:08,522 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:08,522 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:08,523 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1048): stopping wal replay for 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:08,524 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1060): Cleaning up temporary data for 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:08,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T20:42:08,525 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T20:42:08,526 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1093): writing seq id for 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:08,528 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:42:08,529 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1114): Opened 1591a955d2fc691174b77e9b19a3d332; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60412570, jitterRate=-0.0997825562953949}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T20:42:08,529 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:08,530 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1006): Region open journal for 1591a955d2fc691174b77e9b19a3d332: Running coprocessor pre-open hook at 1732653728513Writing region info on filesystem at 1732653728513Initializing all the Stores at 1732653728514 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653728514Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653728514Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653728515 (+1 ms)Cleaning up temporary data from old regions at 1732653728524 (+9 ms)Running coprocessor post-open hooks at 1732653728529 (+5 ms)Region opened successfully at 1732653728530 (+1 ms) 2024-11-26T20:42:08,531 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., pid=95, masterSystemTime=1732653728509 2024-11-26T20:42:08,533 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:08,533 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:08,534 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=1591a955d2fc691174b77e9b19a3d332, regionState=OPEN, openSeqNum=2, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:42:08,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 because future has completed 2024-11-26T20:42:08,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-26T20:42:08,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 in 179 msec 2024-11-26T20:42:08,541 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-26T20:42:08,541 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1591a955d2fc691174b77e9b19a3d332, ASSIGN in 336 msec 2024-11-26T20:42:08,541 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T20:42:08,542 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653728541"}]},"ts":"1732653728541"} 2024-11-26T20:42:08,543 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T20:42:08,544 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T20:42:08,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1620 sec 2024-11-26T20:42:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T20:42:09,534 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:42:09,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f9102f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:09,535 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:42:09,536 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:42:09,537 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:42:09,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:42:09,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:42:09,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73439d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:42:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:42:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:09,539 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:42:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1813b7e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:09,540 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:42:09,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:42:09,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:42:09,542 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:42:09,543 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T20:42:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is cd6be161543e,42359,1732653630959 2024-11-26T20:42:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@64451ca9 2024-11-26T20:42:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T20:42:09,546 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T20:42:09,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T20:42:09,547 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T20:42:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-26T20:42:09,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742165_1341 (size=1097) 2024-11-26T20:42:09,961 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-26T20:42:09,961 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-26T20:42:09,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T20:42:09,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1591a955d2fc691174b77e9b19a3d332, REOPEN/MOVE}] 2024-11-26T20:42:09,966 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1591a955d2fc691174b77e9b19a3d332, REOPEN/MOVE 2024-11-26T20:42:09,967 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=1591a955d2fc691174b77e9b19a3d332, regionState=CLOSING, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:42:09,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1591a955d2fc691174b77e9b19a3d332, REOPEN/MOVE because future has completed 2024-11-26T20:42:09,969 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T20:42:09,969 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795}] 2024-11-26T20:42:10,121 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(122): Close 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:10,121 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T20:42:10,121 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1722): Closing 1591a955d2fc691174b77e9b19a3d332, disabling compactions & flushes 2024-11-26T20:42:10,121 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:10,121 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:10,121 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. after waiting 0 ms 2024-11-26T20:42:10,121 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:10,124 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:42:10,125 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:10,125 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1676): Region close journal for 1591a955d2fc691174b77e9b19a3d332: Waiting for close lock at 1732653730121Running coprocessor pre-close hooks at 1732653730121Disabling compacts and flushes for region at 1732653730121Disabling writes for close at 1732653730121Writing region close event to WAL at 1732653730122 (+1 ms)Running coprocessor post-close hooks at 1732653730125 (+3 ms)Closed at 1732653730125 2024-11-26T20:42:10,125 WARN [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionServer(3299): Not adding moved region record: 1591a955d2fc691174b77e9b19a3d332 to self. 2024-11-26T20:42:10,126 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(157): Closed 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:10,127 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=1591a955d2fc691174b77e9b19a3d332, regionState=CLOSED 2024-11-26T20:42:10,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 because future has completed 2024-11-26T20:42:10,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-26T20:42:10,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; CloseRegionProcedure 1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 in 160 msec 2024-11-26T20:42:10,131 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1591a955d2fc691174b77e9b19a3d332, REOPEN/MOVE; state=CLOSED, location=cd6be161543e,40291,1732653631795; forceNewPlan=false, retain=true 2024-11-26T20:42:10,281 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=1591a955d2fc691174b77e9b19a3d332, regionState=OPENING, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:42:10,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1591a955d2fc691174b77e9b19a3d332, REOPEN/MOVE because future has completed 2024-11-26T20:42:10,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795}] 2024-11-26T20:42:10,437 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:10,437 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7752): Opening region: {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:42:10,437 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:10,437 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:42:10,438 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7794): checking encryption for 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:10,438 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7797): checking classloading for 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:10,439 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:10,439 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:42:10,440 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1591a955d2fc691174b77e9b19a3d332 columnFamilyName A 2024-11-26T20:42:10,441 DEBUG [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:10,441 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] regionserver.HStore(327): Store=1591a955d2fc691174b77e9b19a3d332/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:42:10,441 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:10,442 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:42:10,442 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1591a955d2fc691174b77e9b19a3d332 columnFamilyName B 2024-11-26T20:42:10,442 DEBUG [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:10,442 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] regionserver.HStore(327): Store=1591a955d2fc691174b77e9b19a3d332/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:42:10,442 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:10,443 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:42:10,443 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1591a955d2fc691174b77e9b19a3d332 columnFamilyName C 2024-11-26T20:42:10,443 DEBUG [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:10,443 INFO [StoreOpener-1591a955d2fc691174b77e9b19a3d332-1 {}] regionserver.HStore(327): Store=1591a955d2fc691174b77e9b19a3d332/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:42:10,443 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:10,443 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1038): replaying wal for 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:10,444 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:10,444 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:10,445 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1048): stopping wal replay for 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:10,445 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1060): Cleaning up temporary data for 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:10,446 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T20:42:10,446 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1093): writing seq id for 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:10,447 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1114): Opened 1591a955d2fc691174b77e9b19a3d332; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73882306, jitterRate=0.10093215107917786}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T20:42:10,447 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:10,448 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1006): Region open journal for 1591a955d2fc691174b77e9b19a3d332: Running coprocessor pre-open hook at 1732653730438Writing region info on filesystem at 1732653730438Initializing all the Stores at 1732653730438Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653730438Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653730438Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653730438Cleaning up temporary data from old regions at 1732653730445 (+7 ms)Running coprocessor post-open hooks at 1732653730447 (+2 ms)Region opened successfully at 1732653730448 (+1 ms) 2024-11-26T20:42:10,448 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., pid=100, masterSystemTime=1732653730434 2024-11-26T20:42:10,450 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:10,450 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:10,450 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=1591a955d2fc691174b77e9b19a3d332, regionState=OPEN, openSeqNum=5, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:42:10,451 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 because future has completed 2024-11-26T20:42:10,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=98 2024-11-26T20:42:10,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure 1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 in 169 msec 2024-11-26T20:42:10,454 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-26T20:42:10,454 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1591a955d2fc691174b77e9b19a3d332, REOPEN/MOVE in 487 msec 2024-11-26T20:42:10,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-26T20:42:10,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-11-26T20:42:10,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 909 msec 2024-11-26T20:42:10,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-26T20:42:10,459 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:42:10,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec1958f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,459 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:42:10,459 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:42:10,460 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:42:10,460 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:42:10,460 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:42:10,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e6acec4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:42:10,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:42:10,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:10,461 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:42:10,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@637f8679, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,462 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,462 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:42:10,462 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:42:10,463 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:42:10,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:42:10,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:42:10,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ceacb7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:42:10,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:42:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:10,464 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:42:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58cfe836, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,465 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:42:10,465 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:42:10,466 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:42:10,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:42:10,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:42:10,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5df2b2ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:42:10,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:42:10,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:10,467 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:42:10,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3368d006, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,467 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,467 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:42:10,467 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:42:10,468 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:42:10,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:42:10,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:42:10,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13163c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:42:10,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:42:10,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:10,469 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:42:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3de877b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,470 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,470 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:42:10,470 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:42:10,470 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:42:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:42:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:42:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e99659, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:42:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:42:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:10,471 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:42:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c2d5e31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,472 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,472 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:42:10,472 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:42:10,473 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:42:10,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:42:10,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:42:10,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2246e266, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:42:10,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:42:10,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:10,474 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:42:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f813915, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,475 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,475 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:42:10,475 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:42:10,475 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:42:10,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:42:10,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:42:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46e7bcbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:42:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:42:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:10,476 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:42:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a01e83c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,477 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:42:10,477 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:42:10,478 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:42:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:42:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:42:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48af0fce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:42:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:42:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:10,479 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:42:10,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d151e17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,480 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,480 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:42:10,480 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:42:10,480 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:42:10,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:42:10,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:42:10,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@243519f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:42:10,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:42:10,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:10,481 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:42:10,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c3916f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,484 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,484 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:42:10,484 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:42:10,485 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:42:10,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:42:10,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:42:10,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@376fc8e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:42:10,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:42:10,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:10,486 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:42:10,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dd4ad56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:10,487 DEBUG [Thread-1558 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:42:10,487 DEBUG [Thread-1565 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:42:10,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:42:10,488 DEBUG [Thread-1573 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:42:10,488 DEBUG [Thread-1571 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:42:10,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:42:10,488 DEBUG [Thread-1569 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:42:10,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:42:10,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:42:10,488 DEBUG [Thread-1567 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:42:10,489 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:42:10,489 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:42:10,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:42:10,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:42:10,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:42:10,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:42:10,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:42:10,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:42:10,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:42:10,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:42:10,491 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:42:10,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:42:10,491 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:42:10,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:42:10,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:42:10,492 DEBUG [Thread-1556 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:42:10,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:42:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:42:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:42:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:42:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:42:10,495 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:42:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:42:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:42:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:42:10,498 DEBUG [Thread-1554 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:42:10,499 DEBUG [Thread-1560 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:42:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:42:10,499 DEBUG [Thread-1562 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:42:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:42:10,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:42:10,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:42:10,501 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:42:10,501 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:42:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:42:10,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:42:10,502 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:42:10,502 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:42:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-26T20:42:10,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:42:10,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:42:10,503 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:42:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T20:42:10,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:42:10,504 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:42:10,505 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:42:10,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1591a955d2fc691174b77e9b19a3d332 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T20:42:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:10,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=A 2024-11-26T20:42:10,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:10,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=B 2024-11-26T20:42:10,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:10,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=C 2024-11-26T20:42:10,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:10,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260707b1f4a7c644f8afc22a00897f3c52_1591a955d2fc691174b77e9b19a3d332 is 50, key is test_row_0/A:col10/1732653730504/Put/seqid=0 2024-11-26T20:42:10,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742166_1342 (size=12154) 2024-11-26T20:42:10,553 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:10,556 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260707b1f4a7c644f8afc22a00897f3c52_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260707b1f4a7c644f8afc22a00897f3c52_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:10,559 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/125605ae78c840dd9d31ccc3709f58c7, store: [table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:10,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/125605ae78c840dd9d31ccc3709f58c7 is 175, key is test_row_0/A:col10/1732653730504/Put/seqid=0 2024-11-26T20:42:10,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:10,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653790549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:10,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:10,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653790550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:10,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:10,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:10,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653790552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:10,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:10,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653790553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:10,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653730546, max timestamp=1732653730564Num uniques 859; 2024-11-26T20:42:10,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:42:10,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653730546, max timestamp=1732653730564Num uniques 859; 2024-11-26T20:42:10,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:42:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:10,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653730546, max timestamp=1732653730564Num uniques 859; 2024-11-26T20:42:10,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:42:10,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742167_1343 (size=30955) 2024-11-26T20:42:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T20:42:10,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T20:42:10,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:10,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:10,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:10,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:10,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:10,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653790680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:10,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653790682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:10,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653790683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653790684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:10,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653790692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:10,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:10,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T20:42:10,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653790804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:10,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:10,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:10,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:10,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653790804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:10,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:10,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:10,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653790804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:10,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:10,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653790804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:10,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:10,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:10,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T20:42:10,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:10,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653790814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:10,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:10,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:10,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T20:42:10,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:10,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:10,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:10,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:10,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:10,978 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/125605ae78c840dd9d31ccc3709f58c7 2024-11-26T20:42:11,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/3e931976a35441af8f75a20added0ecb is 50, key is test_row_0/B:col10/1732653730504/Put/seqid=0 2024-11-26T20:42:11,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742168_1344 (size=12001) 2024-11-26T20:42:11,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653791014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:11,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653791014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:11,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:11,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653791023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:11,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:11,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653791024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:11,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:11,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653791034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:11,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T20:42:11,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:11,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:11,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:11,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:11,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:11,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T20:42:11,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T20:42:11,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:11,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:11,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:11,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:11,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:11,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653791323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:11,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:11,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653791323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:11,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653791344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:11,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653791344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:11,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653791354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:11,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:11,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/3e931976a35441af8f75a20added0ecb 2024-11-26T20:42:11,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T20:42:11,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:11,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:11,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:11,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:11,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:11,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/bf8dcab11afb4a6aa8084b0e0dd74309 is 50, key is test_row_0/C:col10/1732653730504/Put/seqid=0 2024-11-26T20:42:11,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742169_1345 (size=12001) 2024-11-26T20:42:11,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/bf8dcab11afb4a6aa8084b0e0dd74309 2024-11-26T20:42:11,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/125605ae78c840dd9d31ccc3709f58c7 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/125605ae78c840dd9d31ccc3709f58c7 2024-11-26T20:42:11,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/125605ae78c840dd9d31ccc3709f58c7, entries=150, sequenceid=15, filesize=30.2 K 2024-11-26T20:42:11,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/3e931976a35441af8f75a20added0ecb as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/3e931976a35441af8f75a20added0ecb 2024-11-26T20:42:11,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/3e931976a35441af8f75a20added0ecb, entries=150, sequenceid=15, filesize=11.7 K 2024-11-26T20:42:11,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/bf8dcab11afb4a6aa8084b0e0dd74309 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/bf8dcab11afb4a6aa8084b0e0dd74309 2024-11-26T20:42:11,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/bf8dcab11afb4a6aa8084b0e0dd74309, entries=150, sequenceid=15, filesize=11.7 K 2024-11-26T20:42:11,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1591a955d2fc691174b77e9b19a3d332 in 960ms, sequenceid=15, compaction requested=false 2024-11-26T20:42:11,468 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-26T20:42:11,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:11,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T20:42:11,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:11,576 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 1591a955d2fc691174b77e9b19a3d332 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-26T20:42:11,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=A 2024-11-26T20:42:11,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:42:11,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=B 2024-11-26T20:42:11,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:42:11,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=C 2024-11-26T20:42:11,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:42:11,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126743efd4f6bf9452e9a5147a2e7016e1b_1591a955d2fc691174b77e9b19a3d332 is 50, key is test_row_0/A:col10/1732653730681/Put/seqid=0 2024-11-26T20:42:11,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742170_1346 (size=12154) 2024-11-26T20:42:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T20:42:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:11,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:11,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653791864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:11,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653791872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:11,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:11,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:11,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653791876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:11,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653791877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:11,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:11,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653791878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:11,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:11,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:11,991 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126743efd4f6bf9452e9a5147a2e7016e1b_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126743efd4f6bf9452e9a5147a2e7016e1b_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:11,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/495332bcfe99442f8827ba863256ac0a, store: [table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:11,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653791984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:11,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/495332bcfe99442f8827ba863256ac0a is 175, key is test_row_0/A:col10/1732653730681/Put/seqid=0 2024-11-26T20:42:11,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653791984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:11,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653791994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:11,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653791995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:12,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742171_1347 (size=30955) 2024-11-26T20:42:12,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653792004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:12,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:12,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:12,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:12,110 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T20:42:12,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653792204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:12,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653792204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:12,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:12,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:12,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:12,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653792205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:12,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:12,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:12,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653792205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:12,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:12,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:12,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:12,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:12,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653792214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:12,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:12,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:12,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:12,406 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/495332bcfe99442f8827ba863256ac0a 2024-11-26T20:42:12,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/0e73a1650e214a918a8129acf203fd59 is 50, key is test_row_0/B:col10/1732653730681/Put/seqid=0 2024-11-26T20:42:12,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742172_1348 (size=12001) 2024-11-26T20:42:12,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653792514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:12,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653792523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:12,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653792524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:12,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:12,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:12,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:12,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:12,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:12,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653792524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:12,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:12,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:12,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:12,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:12,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653792534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T20:42:12,817 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/0e73a1650e214a918a8129acf203fd59 2024-11-26T20:42:12,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/d8a6d650cb924274be9daee0efbbee25 is 50, key is test_row_0/C:col10/1732653730681/Put/seqid=0 2024-11-26T20:42:12,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742173_1349 (size=12001) 2024-11-26T20:42:12,829 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/d8a6d650cb924274be9daee0efbbee25 2024-11-26T20:42:12,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/495332bcfe99442f8827ba863256ac0a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/495332bcfe99442f8827ba863256ac0a 2024-11-26T20:42:12,837 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/495332bcfe99442f8827ba863256ac0a, entries=150, sequenceid=51, filesize=30.2 K 2024-11-26T20:42:12,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/0e73a1650e214a918a8129acf203fd59 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/0e73a1650e214a918a8129acf203fd59 2024-11-26T20:42:12,841 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/0e73a1650e214a918a8129acf203fd59, entries=150, sequenceid=51, filesize=11.7 K 2024-11-26T20:42:12,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/d8a6d650cb924274be9daee0efbbee25 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/d8a6d650cb924274be9daee0efbbee25 2024-11-26T20:42:12,845 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/d8a6d650cb924274be9daee0efbbee25, entries=150, sequenceid=51, filesize=11.7 K 2024-11-26T20:42:12,845 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=60.38 KB/61830 for 1591a955d2fc691174b77e9b19a3d332 in 1269ms, sequenceid=51, compaction requested=false 2024-11-26T20:42:12,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:12,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:12,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-26T20:42:12,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-26T20:42:12,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-26T20:42:12,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3430 sec 2024-11-26T20:42:12,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.3480 sec 2024-11-26T20:42:13,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:13,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1591a955d2fc691174b77e9b19a3d332 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T20:42:13,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=A 2024-11-26T20:42:13,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:13,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=B 2024-11-26T20:42:13,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:13,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=C 2024-11-26T20:42:13,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:13,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112680dfc5cdf69c46e896ee4cd36cdc0bcd_1591a955d2fc691174b77e9b19a3d332 is 50, key is test_row_0/A:col10/1732653733026/Put/seqid=0 2024-11-26T20:42:13,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742174_1350 (size=12154) 2024-11-26T20:42:13,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:13,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653793069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:13,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:13,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653793070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653793077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:13,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:13,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653793078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:13,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:13,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653793080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:13,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:13,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:13,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653793194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:13,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:13,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653793194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:13,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653793194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:13,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:13,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653793194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:13,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:13,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653793204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:13,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653793404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:13,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653793404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:13,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653793405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:13,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653793405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:13,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:13,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:13,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:13,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:13,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653793424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:13,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:13,450 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:13,454 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112680dfc5cdf69c46e896ee4cd36cdc0bcd_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112680dfc5cdf69c46e896ee4cd36cdc0bcd_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:13,455 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/b8576971b5d34391a564c813339c804e, store: [table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:13,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/b8576971b5d34391a564c813339c804e is 175, key is test_row_0/A:col10/1732653733026/Put/seqid=0 2024-11-26T20:42:13,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742175_1351 (size=30955) 2024-11-26T20:42:13,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:13,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653793724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:13,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:13,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653793724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:13,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:13,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:13,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653793724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:13,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653793725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:13,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:13,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653793744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:13,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:13,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:13,859 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/b8576971b5d34391a564c813339c804e 2024-11-26T20:42:13,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/767202d0f4ab42b89a89a9be7ca5ea78 is 50, key is test_row_0/B:col10/1732653733026/Put/seqid=0 2024-11-26T20:42:13,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742176_1352 (size=12001) 2024-11-26T20:42:14,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:14,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653794234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:14,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:14,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:14,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:14,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653794244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:14,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653794244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:14,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:14,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:14,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:14,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:14,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653794244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:14,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:14,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:14,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:14,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653794263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:14,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:14,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:14,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:14,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/767202d0f4ab42b89a89a9be7ca5ea78 2024-11-26T20:42:14,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/f89b4efd23d8434f8172d510a5547060 is 50, key is test_row_0/C:col10/1732653733026/Put/seqid=0 2024-11-26T20:42:14,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742177_1353 (size=12001) 2024-11-26T20:42:14,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/f89b4efd23d8434f8172d510a5547060 2024-11-26T20:42:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T20:42:14,684 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:42:14,684 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:42:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-26T20:42:14,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T20:42:14,687 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:42:14,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/b8576971b5d34391a564c813339c804e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/b8576971b5d34391a564c813339c804e 2024-11-26T20:42:14,688 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:42:14,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:42:14,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/b8576971b5d34391a564c813339c804e, entries=150, sequenceid=64, filesize=30.2 K 2024-11-26T20:42:14,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/767202d0f4ab42b89a89a9be7ca5ea78 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/767202d0f4ab42b89a89a9be7ca5ea78 2024-11-26T20:42:14,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/767202d0f4ab42b89a89a9be7ca5ea78, entries=150, sequenceid=64, filesize=11.7 K 2024-11-26T20:42:14,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/f89b4efd23d8434f8172d510a5547060 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/f89b4efd23d8434f8172d510a5547060 2024-11-26T20:42:14,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/f89b4efd23d8434f8172d510a5547060, entries=150, sequenceid=64, filesize=11.7 K 2024-11-26T20:42:14,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1591a955d2fc691174b77e9b19a3d332 in 1667ms, sequenceid=64, compaction requested=true 2024-11-26T20:42:14,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:14,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1591a955d2fc691174b77e9b19a3d332:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:42:14,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:14,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1591a955d2fc691174b77e9b19a3d332:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:42:14,701 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:14,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:14,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1591a955d2fc691174b77e9b19a3d332:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:42:14,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:14,701 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:14,702 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:14,702 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 1591a955d2fc691174b77e9b19a3d332/B is initiating minor compaction (all files) 2024-11-26T20:42:14,702 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1591a955d2fc691174b77e9b19a3d332/B in TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:14,702 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/3e931976a35441af8f75a20added0ecb, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/0e73a1650e214a918a8129acf203fd59, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/767202d0f4ab42b89a89a9be7ca5ea78] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp, totalSize=35.2 K 2024-11-26T20:42:14,702 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:14,702 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 1591a955d2fc691174b77e9b19a3d332/A is initiating minor compaction (all files) 2024-11-26T20:42:14,702 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1591a955d2fc691174b77e9b19a3d332/A in TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:14,702 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/125605ae78c840dd9d31ccc3709f58c7, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/495332bcfe99442f8827ba863256ac0a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/b8576971b5d34391a564c813339c804e] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp, totalSize=90.7 K 2024-11-26T20:42:14,702 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:14,702 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. files: [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/125605ae78c840dd9d31ccc3709f58c7, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/495332bcfe99442f8827ba863256ac0a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/b8576971b5d34391a564c813339c804e] 2024-11-26T20:42:14,703 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 125605ae78c840dd9d31ccc3709f58c7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732653730498 2024-11-26T20:42:14,703 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e931976a35441af8f75a20added0ecb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732653730498 2024-11-26T20:42:14,703 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e73a1650e214a918a8129acf203fd59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732653730679 2024-11-26T20:42:14,703 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 495332bcfe99442f8827ba863256ac0a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732653730679 2024-11-26T20:42:14,704 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 767202d0f4ab42b89a89a9be7ca5ea78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732653731876 2024-11-26T20:42:14,704 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8576971b5d34391a564c813339c804e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732653731876 2024-11-26T20:42:14,711 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1591a955d2fc691174b77e9b19a3d332#B#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:14,712 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/8b04144eccf7450f8ed4c36c78034496 is 50, key is test_row_0/B:col10/1732653733026/Put/seqid=0 2024-11-26T20:42:14,714 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:14,719 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411262d3cfe9639d349dba89f5033e97b9040_1591a955d2fc691174b77e9b19a3d332 store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:14,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742178_1354 (size=12104) 2024-11-26T20:42:14,721 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411262d3cfe9639d349dba89f5033e97b9040_1591a955d2fc691174b77e9b19a3d332, store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:14,721 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411262d3cfe9639d349dba89f5033e97b9040_1591a955d2fc691174b77e9b19a3d332 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:14,727 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/8b04144eccf7450f8ed4c36c78034496 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/8b04144eccf7450f8ed4c36c78034496 2024-11-26T20:42:14,734 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1591a955d2fc691174b77e9b19a3d332/B of 1591a955d2fc691174b77e9b19a3d332 into 8b04144eccf7450f8ed4c36c78034496(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:14,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742179_1355 (size=4469) 2024-11-26T20:42:14,734 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:14,734 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., storeName=1591a955d2fc691174b77e9b19a3d332/B, priority=13, startTime=1732653734701; duration=0sec 2024-11-26T20:42:14,735 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:14,735 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1591a955d2fc691174b77e9b19a3d332:B 2024-11-26T20:42:14,735 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:14,735 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:14,735 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 1591a955d2fc691174b77e9b19a3d332/C is initiating minor compaction (all files) 2024-11-26T20:42:14,735 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1591a955d2fc691174b77e9b19a3d332/C in TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:14,736 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/bf8dcab11afb4a6aa8084b0e0dd74309, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/d8a6d650cb924274be9daee0efbbee25, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/f89b4efd23d8434f8172d510a5547060] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp, totalSize=35.2 K 2024-11-26T20:42:14,736 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting bf8dcab11afb4a6aa8084b0e0dd74309, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732653730498 2024-11-26T20:42:14,736 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting d8a6d650cb924274be9daee0efbbee25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732653730679 2024-11-26T20:42:14,736 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting f89b4efd23d8434f8172d510a5547060, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732653731876 2024-11-26T20:42:14,748 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1591a955d2fc691174b77e9b19a3d332#C#compaction#308 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:14,748 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/8008a65efd3e44c7a301fa528ac5fe3d is 50, key is test_row_0/C:col10/1732653733026/Put/seqid=0 2024-11-26T20:42:14,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742180_1356 (size=12104) 2024-11-26T20:42:14,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T20:42:14,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T20:42:14,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:14,841 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 1591a955d2fc691174b77e9b19a3d332 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T20:42:14,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=A 2024-11-26T20:42:14,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:14,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=B 2024-11-26T20:42:14,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:14,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=C 2024-11-26T20:42:14,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:14,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b0a6ce931d99426ab4bf935428357a0c_1591a955d2fc691174b77e9b19a3d332 is 50, key is test_row_0/A:col10/1732653733068/Put/seqid=0 2024-11-26T20:42:14,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742181_1357 (size=12154) 2024-11-26T20:42:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T20:42:15,135 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1591a955d2fc691174b77e9b19a3d332#A#compaction#307 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:15,136 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/de05e878e8b44f228349f9a866329735 is 175, key is test_row_0/A:col10/1732653733026/Put/seqid=0 2024-11-26T20:42:15,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742182_1358 (size=31058) 2024-11-26T20:42:15,162 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/8008a65efd3e44c7a301fa528ac5fe3d as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/8008a65efd3e44c7a301fa528ac5fe3d 2024-11-26T20:42:15,167 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1591a955d2fc691174b77e9b19a3d332/C of 1591a955d2fc691174b77e9b19a3d332 into 8008a65efd3e44c7a301fa528ac5fe3d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:15,167 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:15,167 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., storeName=1591a955d2fc691174b77e9b19a3d332/C, priority=13, startTime=1732653734701; duration=0sec 2024-11-26T20:42:15,167 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:15,168 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1591a955d2fc691174b77e9b19a3d332:C 2024-11-26T20:42:15,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:15,256 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b0a6ce931d99426ab4bf935428357a0c_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b0a6ce931d99426ab4bf935428357a0c_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:15,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/392558849c284788b34627809c7e8934, store: [table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:15,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/392558849c284788b34627809c7e8934 is 175, key is test_row_0/A:col10/1732653733068/Put/seqid=0 2024-11-26T20:42:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:15,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:15,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742183_1359 (size=30955) 2024-11-26T20:42:15,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:15,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653795274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:15,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:15,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:15,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653795283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:15,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:15,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653795284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:15,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:15,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:15,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653795284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:15,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:15,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:15,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653795293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:15,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T20:42:15,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653795403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:15,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653795404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:15,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653795404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653795404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:15,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653795414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:15,545 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/de05e878e8b44f228349f9a866329735 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/de05e878e8b44f228349f9a866329735 2024-11-26T20:42:15,549 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1591a955d2fc691174b77e9b19a3d332/A of 1591a955d2fc691174b77e9b19a3d332 into de05e878e8b44f228349f9a866329735(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:15,549 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:15,549 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., storeName=1591a955d2fc691174b77e9b19a3d332/A, priority=13, startTime=1732653734700; duration=0sec 2024-11-26T20:42:15,549 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:15,549 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1591a955d2fc691174b77e9b19a3d332:A 2024-11-26T20:42:15,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653795614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:15,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:15,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653795624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:15,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653795624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:15,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:15,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:15,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653795624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:15,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:15,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:15,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653795633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:15,663 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/392558849c284788b34627809c7e8934 2024-11-26T20:42:15,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/9d461decebb74bc09052708739541fb8 is 50, key is test_row_0/B:col10/1732653733068/Put/seqid=0 2024-11-26T20:42:15,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742184_1360 (size=12001) 2024-11-26T20:42:15,679 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/9d461decebb74bc09052708739541fb8 2024-11-26T20:42:15,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/b69d81d26d564ea0a401a0a274de50ae is 50, key is test_row_0/C:col10/1732653733068/Put/seqid=0 2024-11-26T20:42:15,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742185_1361 (size=12001) 2024-11-26T20:42:15,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T20:42:15,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653795934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:15,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653795944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:15,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653795944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:15,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653795944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:15,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653795944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:16,089 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/b69d81d26d564ea0a401a0a274de50ae 2024-11-26T20:42:16,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/392558849c284788b34627809c7e8934 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/392558849c284788b34627809c7e8934 2024-11-26T20:42:16,097 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/392558849c284788b34627809c7e8934, entries=150, sequenceid=84, filesize=30.2 K 2024-11-26T20:42:16,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/9d461decebb74bc09052708739541fb8 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/9d461decebb74bc09052708739541fb8 2024-11-26T20:42:16,101 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/9d461decebb74bc09052708739541fb8, entries=150, sequenceid=84, filesize=11.7 K 2024-11-26T20:42:16,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/b69d81d26d564ea0a401a0a274de50ae as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/b69d81d26d564ea0a401a0a274de50ae 2024-11-26T20:42:16,107 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/b69d81d26d564ea0a401a0a274de50ae, entries=150, sequenceid=84, filesize=11.7 K 2024-11-26T20:42:16,107 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1591a955d2fc691174b77e9b19a3d332 in 1266ms, sequenceid=84, compaction requested=false 2024-11-26T20:42:16,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:16,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:16,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-26T20:42:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-26T20:42:16,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-26T20:42:16,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4210 sec 2024-11-26T20:42:16,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.4270 sec 2024-11-26T20:42:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:16,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1591a955d2fc691174b77e9b19a3d332 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T20:42:16,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=A 2024-11-26T20:42:16,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:16,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=B 2024-11-26T20:42:16,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:16,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=C 2024-11-26T20:42:16,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:16,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126290c3739a3e74956a062c6f1838bbb36_1591a955d2fc691174b77e9b19a3d332 is 50, key is test_row_0/A:col10/1732653736456/Put/seqid=0 2024-11-26T20:42:16,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742186_1362 (size=14594) 2024-11-26T20:42:16,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653796486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:16,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:16,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:16,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653796493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:16,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:16,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653796496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:16,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653796497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:16,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653796498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:16,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653796604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:16,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653796604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:16,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:16,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:16,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:16,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653796614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:16,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:16,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653796614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:16,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:16,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653796624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:16,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653796814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:16,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:16,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653796824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:16,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:16,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:16,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653796834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:16,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:16,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653796834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:16,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T20:42:16,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:16,845 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:42:16,845 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:42:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-26T20:42:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T20:42:16,847 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:42:16,847 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:42:16,847 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:42:16,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653796844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:16,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:16,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:16,870 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:16,873 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126290c3739a3e74956a062c6f1838bbb36_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126290c3739a3e74956a062c6f1838bbb36_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:16,874 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/b9e92bf5869541e880f08c91e2d71eb3, store: [table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:16,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/b9e92bf5869541e880f08c91e2d71eb3 is 175, key is test_row_0/A:col10/1732653736456/Put/seqid=0 2024-11-26T20:42:16,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742187_1363 (size=39549) 2024-11-26T20:42:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T20:42:16,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T20:42:17,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:17,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:17,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:17,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:17,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:17,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653797134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:17,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T20:42:17,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:17,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:17,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:17,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:17,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:17,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:17,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653797144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:17,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T20:42:17,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653797154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:17,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653797155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:17,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653797164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:17,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:17,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:17,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:17,278 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/b9e92bf5869541e880f08c91e2d71eb3 2024-11-26T20:42:17,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/905c372b364f4ea4ace26bcb80971470 is 50, key is test_row_0/B:col10/1732653736456/Put/seqid=0 2024-11-26T20:42:17,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742188_1364 (size=12001) 2024-11-26T20:42:17,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T20:42:17,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:17,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:17,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:17,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:17,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:17,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T20:42:17,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:17,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:17,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:17,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:17,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:17,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:17,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T20:42:17,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T20:42:17,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:17,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:17,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:17,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:17,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:17,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:17,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:17,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653797654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:17,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:17,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653797673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:17,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:17,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:17,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:17,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653797674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:17,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:17,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:17,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:17,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:17,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653797674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:17,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:17,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:17,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:17,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653797674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:17,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:17,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:17,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:17,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:17,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:17,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/905c372b364f4ea4ace26bcb80971470 2024-11-26T20:42:17,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/91a05d5738864992a24e76cdb298c2a8 is 50, key is test_row_0/C:col10/1732653736456/Put/seqid=0 2024-11-26T20:42:17,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742189_1365 (size=12001) 2024-11-26T20:42:17,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/91a05d5738864992a24e76cdb298c2a8 2024-11-26T20:42:17,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/b9e92bf5869541e880f08c91e2d71eb3 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/b9e92bf5869541e880f08c91e2d71eb3 2024-11-26T20:42:17,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/b9e92bf5869541e880f08c91e2d71eb3, entries=200, sequenceid=100, filesize=38.6 K 2024-11-26T20:42:17,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/905c372b364f4ea4ace26bcb80971470 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/905c372b364f4ea4ace26bcb80971470 2024-11-26T20:42:17,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/905c372b364f4ea4ace26bcb80971470, entries=150, sequenceid=100, filesize=11.7 K 2024-11-26T20:42:17,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/91a05d5738864992a24e76cdb298c2a8 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/91a05d5738864992a24e76cdb298c2a8 2024-11-26T20:42:17,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/91a05d5738864992a24e76cdb298c2a8, entries=150, sequenceid=100, filesize=11.7 K 2024-11-26T20:42:17,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 1591a955d2fc691174b77e9b19a3d332 in 1262ms, sequenceid=100, compaction requested=true 2024-11-26T20:42:17,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:17,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1591a955d2fc691174b77e9b19a3d332:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:42:17,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:17,720 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:17,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1591a955d2fc691174b77e9b19a3d332:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:42:17,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:17,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1591a955d2fc691174b77e9b19a3d332:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:42:17,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T20:42:17,720 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:17,721 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:17,721 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 1591a955d2fc691174b77e9b19a3d332/A is initiating minor compaction (all files) 2024-11-26T20:42:17,721 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1591a955d2fc691174b77e9b19a3d332/A in TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:17,721 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/de05e878e8b44f228349f9a866329735, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/392558849c284788b34627809c7e8934, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/b9e92bf5869541e880f08c91e2d71eb3] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp, totalSize=99.2 K 2024-11-26T20:42:17,721 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:17,721 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. files: [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/de05e878e8b44f228349f9a866329735, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/392558849c284788b34627809c7e8934, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/b9e92bf5869541e880f08c91e2d71eb3] 2024-11-26T20:42:17,722 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting de05e878e8b44f228349f9a866329735, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732653731876 2024-11-26T20:42:17,722 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 392558849c284788b34627809c7e8934, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732653733063 2024-11-26T20:42:17,722 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9e92bf5869541e880f08c91e2d71eb3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732653735282 2024-11-26T20:42:17,724 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:17,724 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 1591a955d2fc691174b77e9b19a3d332/B is initiating minor compaction (all files) 2024-11-26T20:42:17,724 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1591a955d2fc691174b77e9b19a3d332/B in TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:17,724 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/8b04144eccf7450f8ed4c36c78034496, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/9d461decebb74bc09052708739541fb8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/905c372b364f4ea4ace26bcb80971470] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp, totalSize=35.3 K 2024-11-26T20:42:17,724 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b04144eccf7450f8ed4c36c78034496, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732653731876 2024-11-26T20:42:17,727 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d461decebb74bc09052708739541fb8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732653733063 2024-11-26T20:42:17,728 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 905c372b364f4ea4ace26bcb80971470, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732653735282 2024-11-26T20:42:17,731 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:17,737 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126038221a4feeb488fa42a960ff801a410_1591a955d2fc691174b77e9b19a3d332 store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:17,739 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126038221a4feeb488fa42a960ff801a410_1591a955d2fc691174b77e9b19a3d332, store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:17,739 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126038221a4feeb488fa42a960ff801a410_1591a955d2fc691174b77e9b19a3d332 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:17,742 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1591a955d2fc691174b77e9b19a3d332#B#compaction#316 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:17,742 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/b579b8b978224134b7951bc6a2b2f6b4 is 50, key is test_row_0/B:col10/1732653736456/Put/seqid=0 2024-11-26T20:42:17,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742190_1366 (size=4469) 2024-11-26T20:42:17,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742191_1367 (size=12207) 2024-11-26T20:42:17,759 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/b579b8b978224134b7951bc6a2b2f6b4 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/b579b8b978224134b7951bc6a2b2f6b4 2024-11-26T20:42:17,764 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1591a955d2fc691174b77e9b19a3d332/B of 1591a955d2fc691174b77e9b19a3d332 into b579b8b978224134b7951bc6a2b2f6b4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:17,764 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:17,764 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., storeName=1591a955d2fc691174b77e9b19a3d332/B, priority=13, startTime=1732653737720; duration=0sec 2024-11-26T20:42:17,764 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:17,764 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1591a955d2fc691174b77e9b19a3d332:B 2024-11-26T20:42:17,765 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:17,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T20:42:17,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:17,765 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:17,766 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 1591a955d2fc691174b77e9b19a3d332/C is initiating minor compaction (all files) 2024-11-26T20:42:17,766 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 1591a955d2fc691174b77e9b19a3d332 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T20:42:17,766 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1591a955d2fc691174b77e9b19a3d332/C in TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:17,766 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/8008a65efd3e44c7a301fa528ac5fe3d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/b69d81d26d564ea0a401a0a274de50ae, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/91a05d5738864992a24e76cdb298c2a8] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp, totalSize=35.3 K 2024-11-26T20:42:17,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=A 2024-11-26T20:42:17,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:17,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=B 2024-11-26T20:42:17,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:17,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=C 2024-11-26T20:42:17,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:17,766 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 8008a65efd3e44c7a301fa528ac5fe3d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732653731876 2024-11-26T20:42:17,766 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting b69d81d26d564ea0a401a0a274de50ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732653733063 2024-11-26T20:42:17,767 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 91a05d5738864992a24e76cdb298c2a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732653735282 2024-11-26T20:42:17,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126cc14787ce0cf4046be864b6264163701_1591a955d2fc691174b77e9b19a3d332 is 50, key is test_row_0/A:col10/1732653736496/Put/seqid=0 2024-11-26T20:42:17,778 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1591a955d2fc691174b77e9b19a3d332#C#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:17,778 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/3061bdaa8aef49acaa273f235283294a is 50, key is test_row_0/C:col10/1732653736456/Put/seqid=0 2024-11-26T20:42:17,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742192_1368 (size=12154) 2024-11-26T20:42:17,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742193_1369 (size=12207) 2024-11-26T20:42:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T20:42:18,147 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1591a955d2fc691174b77e9b19a3d332#A#compaction#315 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:18,148 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/04d4dffa2f9642c4ba7a145b6b669eb5 is 175, key is test_row_0/A:col10/1732653736456/Put/seqid=0 2024-11-26T20:42:18,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742194_1370 (size=31161) 2024-11-26T20:42:18,162 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/04d4dffa2f9642c4ba7a145b6b669eb5 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/04d4dffa2f9642c4ba7a145b6b669eb5 2024-11-26T20:42:18,166 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1591a955d2fc691174b77e9b19a3d332/A of 1591a955d2fc691174b77e9b19a3d332 into 04d4dffa2f9642c4ba7a145b6b669eb5(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:18,166 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:18,166 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., storeName=1591a955d2fc691174b77e9b19a3d332/A, priority=13, startTime=1732653737720; duration=0sec 2024-11-26T20:42:18,167 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:18,167 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1591a955d2fc691174b77e9b19a3d332:A 2024-11-26T20:42:18,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:18,185 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126cc14787ce0cf4046be864b6264163701_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126cc14787ce0cf4046be864b6264163701_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:18,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/3e083b7a070d4b78aed72c14caf5b35c, store: [table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:18,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/3e083b7a070d4b78aed72c14caf5b35c is 175, key is test_row_0/A:col10/1732653736496/Put/seqid=0 2024-11-26T20:42:18,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742195_1371 (size=30955) 2024-11-26T20:42:18,199 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/3061bdaa8aef49acaa273f235283294a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/3061bdaa8aef49acaa273f235283294a 2024-11-26T20:42:18,202 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1591a955d2fc691174b77e9b19a3d332/C of 1591a955d2fc691174b77e9b19a3d332 into 3061bdaa8aef49acaa273f235283294a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:18,202 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:18,202 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., storeName=1591a955d2fc691174b77e9b19a3d332/C, priority=13, startTime=1732653737720; duration=0sec 2024-11-26T20:42:18,202 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:18,202 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1591a955d2fc691174b77e9b19a3d332:C 2024-11-26T20:42:18,593 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/3e083b7a070d4b78aed72c14caf5b35c 2024-11-26T20:42:18,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/6827d9da919648e588dfe38d25707c04 is 50, key is test_row_0/B:col10/1732653736496/Put/seqid=0 2024-11-26T20:42:18,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742196_1372 (size=12001) 2024-11-26T20:42:18,614 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/6827d9da919648e588dfe38d25707c04 2024-11-26T20:42:18,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/02fed51a18d5483796765a05d612f51f is 50, key is test_row_0/C:col10/1732653736496/Put/seqid=0 2024-11-26T20:42:18,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742197_1373 (size=12001) 2024-11-26T20:42:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:18,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:18,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653798693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:18,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653798694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:18,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:18,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:18,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:18,703 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 1591a955d2fc691174b77e9b19a3d332 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:42:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:18,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653798701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:18,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653798702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:18,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653798703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:18,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653798813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:18,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653798813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:18,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653798814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:18,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653798814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T20:42:19,030 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/02fed51a18d5483796765a05d612f51f 2024-11-26T20:42:19,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653799023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:19,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653799024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:19,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653799024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653799024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:19,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/3e083b7a070d4b78aed72c14caf5b35c as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/3e083b7a070d4b78aed72c14caf5b35c 2024-11-26T20:42:19,039 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/3e083b7a070d4b78aed72c14caf5b35c, entries=150, sequenceid=120, filesize=30.2 K 2024-11-26T20:42:19,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/6827d9da919648e588dfe38d25707c04 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/6827d9da919648e588dfe38d25707c04 2024-11-26T20:42:19,043 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/6827d9da919648e588dfe38d25707c04, entries=150, sequenceid=120, filesize=11.7 K 2024-11-26T20:42:19,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/02fed51a18d5483796765a05d612f51f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/02fed51a18d5483796765a05d612f51f 2024-11-26T20:42:19,047 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/02fed51a18d5483796765a05d612f51f, entries=150, sequenceid=120, filesize=11.7 K 2024-11-26T20:42:19,048 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1591a955d2fc691174b77e9b19a3d332 in 1283ms, sequenceid=120, compaction requested=false 2024-11-26T20:42:19,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:19,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:19,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-26T20:42:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-26T20:42:19,052 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-26T20:42:19,052 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2030 sec 2024-11-26T20:42:19,053 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.2070 sec 2024-11-26T20:42:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:19,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1591a955d2fc691174b77e9b19a3d332 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T20:42:19,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=A 2024-11-26T20:42:19,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:19,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=B 2024-11-26T20:42:19,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:19,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=C 2024-11-26T20:42:19,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:19,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126819efedfa3754f5c97b1b165197d87ff_1591a955d2fc691174b77e9b19a3d332 is 50, key is test_row_0/A:col10/1732653739340/Put/seqid=0 2024-11-26T20:42:19,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742198_1374 (size=12304) 2024-11-26T20:42:19,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653799383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:19,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653799385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:19,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:19,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:19,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653799386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:19,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:19,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653799386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:19,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:19,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:19,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:19,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653799504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:19,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:19,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653799504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:19,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:19,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:19,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:19,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653799514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:19,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:19,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653799514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:19,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:19,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:19,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:19,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653799714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:19,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:19,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653799714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:19,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653799734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:19,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:19,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653799735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:19,756 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:19,760 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126819efedfa3754f5c97b1b165197d87ff_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126819efedfa3754f5c97b1b165197d87ff_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:19,761 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/e500e20efe9a4286931c174cac4acb77, store: [table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:19,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/e500e20efe9a4286931c174cac4acb77 is 175, key is test_row_0/A:col10/1732653739340/Put/seqid=0 2024-11-26T20:42:19,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742199_1375 (size=31105) 2024-11-26T20:42:19,793 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/e500e20efe9a4286931c174cac4acb77 2024-11-26T20:42:19,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/442daf682217497ca81b5b3a447b4457 is 50, key is test_row_0/B:col10/1732653739340/Put/seqid=0 2024-11-26T20:42:19,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742200_1376 (size=12151) 2024-11-26T20:42:20,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:20,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653800034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:20,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:20,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653800034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:20,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:20,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653800054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:20,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653800054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:20,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:20,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/442daf682217497ca81b5b3a447b4457 2024-11-26T20:42:20,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/9f53df94694848edb313e406892fc029 is 50, key is test_row_0/C:col10/1732653739340/Put/seqid=0 2024-11-26T20:42:20,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742201_1377 (size=12151) 2024-11-26T20:42:20,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/9f53df94694848edb313e406892fc029 2024-11-26T20:42:20,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/e500e20efe9a4286931c174cac4acb77 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/e500e20efe9a4286931c174cac4acb77 2024-11-26T20:42:20,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/e500e20efe9a4286931c174cac4acb77, entries=150, sequenceid=136, filesize=30.4 K 2024-11-26T20:42:20,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/442daf682217497ca81b5b3a447b4457 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/442daf682217497ca81b5b3a447b4457 2024-11-26T20:42:20,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/442daf682217497ca81b5b3a447b4457, entries=150, sequenceid=136, filesize=11.9 K 2024-11-26T20:42:20,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/9f53df94694848edb313e406892fc029 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/9f53df94694848edb313e406892fc029 2024-11-26T20:42:20,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/9f53df94694848edb313e406892fc029, entries=150, sequenceid=136, filesize=11.9 K 2024-11-26T20:42:20,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 1591a955d2fc691174b77e9b19a3d332 in 902ms, sequenceid=136, compaction requested=true 2024-11-26T20:42:20,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:20,244 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:20,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1591a955d2fc691174b77e9b19a3d332:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:42:20,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:20,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1591a955d2fc691174b77e9b19a3d332:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:42:20,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:20,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1591a955d2fc691174b77e9b19a3d332:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:42:20,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:20,244 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:20,244 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:20,244 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 1591a955d2fc691174b77e9b19a3d332/A is initiating minor compaction (all files) 2024-11-26T20:42:20,245 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1591a955d2fc691174b77e9b19a3d332/A in TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:20,245 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/04d4dffa2f9642c4ba7a145b6b669eb5, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/3e083b7a070d4b78aed72c14caf5b35c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/e500e20efe9a4286931c174cac4acb77] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp, totalSize=91.0 K 2024-11-26T20:42:20,245 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:20,245 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. files: [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/04d4dffa2f9642c4ba7a145b6b669eb5, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/3e083b7a070d4b78aed72c14caf5b35c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/e500e20efe9a4286931c174cac4acb77] 2024-11-26T20:42:20,245 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04d4dffa2f9642c4ba7a145b6b669eb5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732653735282 2024-11-26T20:42:20,245 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:20,246 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 1591a955d2fc691174b77e9b19a3d332/B is initiating minor compaction (all files) 2024-11-26T20:42:20,246 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1591a955d2fc691174b77e9b19a3d332/B in TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:20,246 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/b579b8b978224134b7951bc6a2b2f6b4, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/6827d9da919648e588dfe38d25707c04, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/442daf682217497ca81b5b3a447b4457] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp, totalSize=35.5 K 2024-11-26T20:42:20,247 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting b579b8b978224134b7951bc6a2b2f6b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732653735282 2024-11-26T20:42:20,247 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e083b7a070d4b78aed72c14caf5b35c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732653736495 2024-11-26T20:42:20,247 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting e500e20efe9a4286931c174cac4acb77, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732653738690 2024-11-26T20:42:20,247 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 6827d9da919648e588dfe38d25707c04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732653736495 2024-11-26T20:42:20,247 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 442daf682217497ca81b5b3a447b4457, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732653738690 2024-11-26T20:42:20,252 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:20,253 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1591a955d2fc691174b77e9b19a3d332#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:20,254 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/78ca5409b1ac46dda9db2249b243fc62 is 50, key is test_row_0/B:col10/1732653739340/Put/seqid=0 2024-11-26T20:42:20,254 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411260d27dcd3231c4b2185449e2235d06507_1591a955d2fc691174b77e9b19a3d332 store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:20,256 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411260d27dcd3231c4b2185449e2235d06507_1591a955d2fc691174b77e9b19a3d332, store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:20,256 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260d27dcd3231c4b2185449e2235d06507_1591a955d2fc691174b77e9b19a3d332 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:20,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742202_1378 (size=12459) 2024-11-26T20:42:20,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742203_1379 (size=4469) 2024-11-26T20:42:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:20,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1591a955d2fc691174b77e9b19a3d332 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T20:42:20,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=A 2024-11-26T20:42:20,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:20,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=B 2024-11-26T20:42:20,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:20,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=C 2024-11-26T20:42:20,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:20,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112668473e32afac4ebeb93e73b02e499418_1591a955d2fc691174b77e9b19a3d332 is 50, key is test_row_0/A:col10/1732653739384/Put/seqid=0 2024-11-26T20:42:20,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742204_1380 (size=14794) 2024-11-26T20:42:20,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653800589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:20,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653800590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:20,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:20,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653800592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:20,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:20,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653800593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:20,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:20,663 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1591a955d2fc691174b77e9b19a3d332#A#compaction#325 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:20,664 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/000e32a8b573413db27e32e338f3da78 is 175, key is test_row_0/A:col10/1732653739340/Put/seqid=0 2024-11-26T20:42:20,667 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/78ca5409b1ac46dda9db2249b243fc62 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/78ca5409b1ac46dda9db2249b243fc62 2024-11-26T20:42:20,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742205_1381 (size=31413) 2024-11-26T20:42:20,673 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1591a955d2fc691174b77e9b19a3d332/B of 1591a955d2fc691174b77e9b19a3d332 into 78ca5409b1ac46dda9db2249b243fc62(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:20,673 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:20,673 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., storeName=1591a955d2fc691174b77e9b19a3d332/B, priority=13, startTime=1732653740244; duration=0sec 2024-11-26T20:42:20,673 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:20,673 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1591a955d2fc691174b77e9b19a3d332:B 2024-11-26T20:42:20,673 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:20,674 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:20,674 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 1591a955d2fc691174b77e9b19a3d332/C is initiating minor compaction (all files) 2024-11-26T20:42:20,674 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1591a955d2fc691174b77e9b19a3d332/C in TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:20,674 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/3061bdaa8aef49acaa273f235283294a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/02fed51a18d5483796765a05d612f51f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/9f53df94694848edb313e406892fc029] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp, totalSize=35.5 K 2024-11-26T20:42:20,675 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 3061bdaa8aef49acaa273f235283294a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732653735282 2024-11-26T20:42:20,675 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 02fed51a18d5483796765a05d612f51f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732653736495 2024-11-26T20:42:20,675 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f53df94694848edb313e406892fc029, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732653738690 2024-11-26T20:42:20,682 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1591a955d2fc691174b77e9b19a3d332#C#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:20,683 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/9c70fd0c872748f0ac75fe389735bb53 is 50, key is test_row_0/C:col10/1732653739340/Put/seqid=0 2024-11-26T20:42:20,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742206_1382 (size=12459) 2024-11-26T20:42:20,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653800714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:20,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653800714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:20,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653800714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:20,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:20,723 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 1591a955d2fc691174b77e9b19a3d332 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:42:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653800714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:20,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653800714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:20,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653800934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:20,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653800934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:20,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:20,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653800934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:20,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653800934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:20,973 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:20,977 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112668473e32afac4ebeb93e73b02e499418_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112668473e32afac4ebeb93e73b02e499418_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:20,977 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/ef1100f3aa10427c9b06cb8470c833c4, store: [table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:20,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/ef1100f3aa10427c9b06cb8470c833c4 is 175, key is test_row_0/A:col10/1732653739384/Put/seqid=0 2024-11-26T20:42:20,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742207_1383 (size=39749) 2024-11-26T20:42:20,985 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/ef1100f3aa10427c9b06cb8470c833c4 2024-11-26T20:42:20,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/3644c498a1044bb294494362661c95d6 is 50, key is test_row_0/B:col10/1732653739384/Put/seqid=0 2024-11-26T20:42:21,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742208_1384 (size=12151) 2024-11-26T20:42:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T20:42:21,014 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:42:21,014 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:42:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-26T20:42:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T20:42:21,016 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:42:21,017 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:42:21,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:42:21,073 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/000e32a8b573413db27e32e338f3da78 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/000e32a8b573413db27e32e338f3da78 2024-11-26T20:42:21,077 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1591a955d2fc691174b77e9b19a3d332/A of 1591a955d2fc691174b77e9b19a3d332 into 000e32a8b573413db27e32e338f3da78(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:21,077 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:21,077 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., storeName=1591a955d2fc691174b77e9b19a3d332/A, priority=13, startTime=1732653740243; duration=0sec 2024-11-26T20:42:21,077 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:21,077 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1591a955d2fc691174b77e9b19a3d332:A 2024-11-26T20:42:21,094 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/9c70fd0c872748f0ac75fe389735bb53 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/9c70fd0c872748f0ac75fe389735bb53 2024-11-26T20:42:21,098 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1591a955d2fc691174b77e9b19a3d332/C of 1591a955d2fc691174b77e9b19a3d332 into 9c70fd0c872748f0ac75fe389735bb53(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:21,098 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:21,098 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., storeName=1591a955d2fc691174b77e9b19a3d332/C, priority=13, startTime=1732653740244; duration=0sec 2024-11-26T20:42:21,098 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:21,098 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1591a955d2fc691174b77e9b19a3d332:C 2024-11-26T20:42:21,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T20:42:21,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T20:42:21,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:21,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:21,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:21,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:21,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:21,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:21,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:21,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653801244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:21,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:21,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653801244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:21,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:21,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:21,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653801245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:21,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:21,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:21,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:21,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653801254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:21,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:21,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T20:42:21,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:21,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:21,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:21,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:21,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:21,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:21,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T20:42:21,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/3644c498a1044bb294494362661c95d6 2024-11-26T20:42:21,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/c649bf8a41d3472d8d181d11a3ff701d is 50, key is test_row_0/C:col10/1732653739384/Put/seqid=0 2024-11-26T20:42:21,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742209_1385 (size=12151) 2024-11-26T20:42:21,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/c649bf8a41d3472d8d181d11a3ff701d 2024-11-26T20:42:21,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/ef1100f3aa10427c9b06cb8470c833c4 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ef1100f3aa10427c9b06cb8470c833c4 2024-11-26T20:42:21,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ef1100f3aa10427c9b06cb8470c833c4, entries=200, sequenceid=156, filesize=38.8 K 2024-11-26T20:42:21,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/3644c498a1044bb294494362661c95d6 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/3644c498a1044bb294494362661c95d6 2024-11-26T20:42:21,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/3644c498a1044bb294494362661c95d6, entries=150, sequenceid=156, filesize=11.9 K 2024-11-26T20:42:21,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/c649bf8a41d3472d8d181d11a3ff701d as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/c649bf8a41d3472d8d181d11a3ff701d 2024-11-26T20:42:21,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/c649bf8a41d3472d8d181d11a3ff701d, entries=150, sequenceid=156, filesize=11.9 K 2024-11-26T20:42:21,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1591a955d2fc691174b77e9b19a3d332 in 876ms, sequenceid=156, compaction requested=false 2024-11-26T20:42:21,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:21,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T20:42:21,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:21,475 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 1591a955d2fc691174b77e9b19a3d332 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T20:42:21,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=A 2024-11-26T20:42:21,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:21,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=B 2024-11-26T20:42:21,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:21,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=C 2024-11-26T20:42:21,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:21,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112673887274496340318ea27fadcd915a6c_1591a955d2fc691174b77e9b19a3d332 is 50, key is test_row_0/A:col10/1732653740591/Put/seqid=0 2024-11-26T20:42:21,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742210_1386 (size=12304) 2024-11-26T20:42:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T20:42:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:21,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:21,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653801801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:21,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653801802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:21,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653801811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:21,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653801812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:21,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:21,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:21,894 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112673887274496340318ea27fadcd915a6c_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112673887274496340318ea27fadcd915a6c_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:21,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/4c3ad95fb7544cc389cfe6581d50b4c9, store: [table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:21,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/4c3ad95fb7544cc389cfe6581d50b4c9 is 175, key is test_row_0/A:col10/1732653740591/Put/seqid=0 2024-11-26T20:42:21,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742211_1387 (size=31105) 2024-11-26T20:42:21,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653801924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:21,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653801924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:21,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653801924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:21,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653801924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:22,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653802134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:22,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653802134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:22,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653802134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:22,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653802134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T20:42:22,300 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/4c3ad95fb7544cc389cfe6581d50b4c9 2024-11-26T20:42:22,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/8df528c8e59b4ecf8bd541f20d48934f is 50, key is test_row_0/B:col10/1732653740591/Put/seqid=0 2024-11-26T20:42:22,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742212_1388 (size=12151) 2024-11-26T20:42:22,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653802444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:22,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:22,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653802444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:22,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:22,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653802454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:22,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653802454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:22,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:22,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:22,719 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/8df528c8e59b4ecf8bd541f20d48934f 2024-11-26T20:42:22,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/2a8597f6e5534f1c972e29021bc07f63 is 50, key is test_row_0/C:col10/1732653740591/Put/seqid=0 2024-11-26T20:42:22,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742213_1389 (size=12151) 2024-11-26T20:42:22,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653802954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:22,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653802963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:22,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:22,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653802963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:22,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:22,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653802974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:22,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:22,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:23,130 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/2a8597f6e5534f1c972e29021bc07f63 2024-11-26T20:42:23,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/4c3ad95fb7544cc389cfe6581d50b4c9 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/4c3ad95fb7544cc389cfe6581d50b4c9 2024-11-26T20:42:23,139 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/4c3ad95fb7544cc389cfe6581d50b4c9, entries=150, sequenceid=171, filesize=30.4 K 2024-11-26T20:42:23,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/8df528c8e59b4ecf8bd541f20d48934f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/8df528c8e59b4ecf8bd541f20d48934f 2024-11-26T20:42:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,144 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/8df528c8e59b4ecf8bd541f20d48934f, entries=150, sequenceid=171, filesize=11.9 K 2024-11-26T20:42:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/2a8597f6e5534f1c972e29021bc07f63 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/2a8597f6e5534f1c972e29021bc07f63 2024-11-26T20:42:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,150 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/2a8597f6e5534f1c972e29021bc07f63, entries=150, sequenceid=171, filesize=11.9 K 2024-11-26T20:42:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,151 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1591a955d2fc691174b77e9b19a3d332 in 1676ms, sequenceid=171, compaction requested=true 2024-11-26T20:42:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:23,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:23,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-26T20:42:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-26T20:42:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-26T20:42:23,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1350 sec 2024-11-26T20:42:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,163 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.1480 sec 2024-11-26T20:42:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T20:42:23,164 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:42:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,165 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:42:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-26T20:42:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,167 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:42:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T20:42:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,168 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:42:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:42:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T20:42:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-26T20:42:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:23,321 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 1591a955d2fc691174b77e9b19a3d332 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T20:42:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=A 2024-11-26T20:42:23,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:23,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=B 2024-11-26T20:42:23,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:23,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=C 2024-11-26T20:42:23,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126285abf256a0e490aa4b62d4273ec3886_1591a955d2fc691174b77e9b19a3d332 is 50, key is test_row_0/A:col10/1732653741810/Put/seqid=0 2024-11-26T20:42:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742214_1390 (size=9814) 2024-11-26T20:42:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,346 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126285abf256a0e490aa4b62d4273ec3886_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126285abf256a0e490aa4b62d4273ec3886_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/135d141f7b7746688eacc56bd80c212d, store: [table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/135d141f7b7746688eacc56bd80c212d is 175, key is test_row_0/A:col10/1732653741810/Put/seqid=0 2024-11-26T20:42:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742215_1391 (size=22461) 2024-11-26T20:42:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T20:42:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,756 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/135d141f7b7746688eacc56bd80c212d 2024-11-26T20:42:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/811a914ce45b49dc8f05d2e8d4d0f686 is 50, key is test_row_0/B:col10/1732653741810/Put/seqid=0 2024-11-26T20:42:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742216_1392 (size=9757) 2024-11-26T20:42:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,777 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/811a914ce45b49dc8f05d2e8d4d0f686 2024-11-26T20:42:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/a88070697f764ac2a46929c2ff88e4f8 is 50, key is test_row_0/C:col10/1732653741810/Put/seqid=0 2024-11-26T20:42:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742217_1393 (size=9757) 2024-11-26T20:42:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T20:42:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:24,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653804041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:24,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653804042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:24,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653804043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:24,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653804053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:24,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:24,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653804164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:24,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653804164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:24,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653804164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:24,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653804174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:24,191 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/a88070697f764ac2a46929c2ff88e4f8 2024-11-26T20:42:24,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/135d141f7b7746688eacc56bd80c212d as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/135d141f7b7746688eacc56bd80c212d 2024-11-26T20:42:24,199 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/135d141f7b7746688eacc56bd80c212d, entries=100, sequenceid=191, filesize=21.9 K 2024-11-26T20:42:24,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/811a914ce45b49dc8f05d2e8d4d0f686 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/811a914ce45b49dc8f05d2e8d4d0f686 2024-11-26T20:42:24,204 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/811a914ce45b49dc8f05d2e8d4d0f686, entries=100, sequenceid=191, filesize=9.5 K 2024-11-26T20:42:24,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/a88070697f764ac2a46929c2ff88e4f8 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/a88070697f764ac2a46929c2ff88e4f8 2024-11-26T20:42:24,209 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/a88070697f764ac2a46929c2ff88e4f8, entries=100, sequenceid=191, filesize=9.5 K 2024-11-26T20:42:24,210 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1591a955d2fc691174b77e9b19a3d332 in 888ms, sequenceid=191, compaction requested=true 2024-11-26T20:42:24,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:24,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:24,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-26T20:42:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-26T20:42:24,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-26T20:42:24,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0430 sec 2024-11-26T20:42:24,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.0490 sec 2024-11-26T20:42:24,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T20:42:24,304 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:42:24,304 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:42:24,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-26T20:42:24,306 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:42:24,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T20:42:24,307 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:42:24,307 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:42:24,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:24,391 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1591a955d2fc691174b77e9b19a3d332 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T20:42:24,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=A 2024-11-26T20:42:24,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:24,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=B 2024-11-26T20:42:24,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:24,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=C 2024-11-26T20:42:24,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:24,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411268a1fd9d0301c4e879e26f4b15e64acaf_1591a955d2fc691174b77e9b19a3d332 is 50, key is test_row_0/A:col10/1732653744042/Put/seqid=0 2024-11-26T20:42:24,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742218_1394 (size=14794) 2024-11-26T20:42:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T20:42:24,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653804438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:24,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:24,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653804438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:24,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:24,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653804440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:24,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653804444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:24,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:24,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:24,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:24,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T20:42:24,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:24,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:24,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:24,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:24,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:24,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653804563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:24,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653804563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:24,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653804563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:24,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653804564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:24,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T20:42:24,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:24,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:24,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:24,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:24,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T20:42:24,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:24,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T20:42:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50586 deadline: 1732653804754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:24,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:24,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:24,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:24,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:24,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:24,765 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 1591a955d2fc691174b77e9b19a3d332 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8267 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:42:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:24,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653804773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:24,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653804774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:24,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653804784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:24,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653804784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:24,807 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:24,811 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411268a1fd9d0301c4e879e26f4b15e64acaf_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411268a1fd9d0301c4e879e26f4b15e64acaf_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:24,812 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/c5c154b993b44d889e1182f1738ca4c5, store: [table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:24,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/c5c154b993b44d889e1182f1738ca4c5 is 175, key is test_row_0/A:col10/1732653744042/Put/seqid=0 2024-11-26T20:42:24,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742219_1395 (size=39749) 2024-11-26T20:42:24,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T20:42:24,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:24,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:24,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:24,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:24,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T20:42:25,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T20:42:25,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:25,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:25,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:25,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:25,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:25,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:25,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653805094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:25,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:25,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653805094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:25,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653805104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:25,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653805104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:25,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:25,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:25,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:25,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:25,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:25,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:25,216 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/c5c154b993b44d889e1182f1738ca4c5 2024-11-26T20:42:25,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/717ab9845bb6403c9f7ba94ffaf231b3 is 50, key is test_row_0/B:col10/1732653744042/Put/seqid=0 2024-11-26T20:42:25,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T20:42:25,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:25,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:25,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:25,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:25,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:25,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:25,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742220_1396 (size=12151) 2024-11-26T20:42:25,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T20:42:25,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:25,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:25,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:25,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:25,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:25,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:25,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T20:42:25,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T20:42:25,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:25,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:25,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:25,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:25,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:25,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:25,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653805603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:25,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:25,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:25,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:25,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653805604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:25,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:25,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:25,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:25,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/717ab9845bb6403c9f7ba94ffaf231b3 2024-11-26T20:42:25,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653805624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:25,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:25,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:25,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653805624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:25,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:25,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:25,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:25,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:25,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/4433e6e758c34c56a25359a05cf585cb is 50, key is test_row_0/C:col10/1732653744042/Put/seqid=0 2024-11-26T20:42:25,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742221_1397 (size=12151) 2024-11-26T20:42:25,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T20:42:25,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:25,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:25,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:25,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:25,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:25,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T20:42:25,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:25,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:25,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:25,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:25,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:25,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:25,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T20:42:25,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:25,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:25,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:25,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:25,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:26,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/4433e6e758c34c56a25359a05cf585cb 2024-11-26T20:42:26,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/c5c154b993b44d889e1182f1738ca4c5 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/c5c154b993b44d889e1182f1738ca4c5 2024-11-26T20:42:26,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/c5c154b993b44d889e1182f1738ca4c5, entries=200, sequenceid=204, filesize=38.8 K 2024-11-26T20:42:26,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/717ab9845bb6403c9f7ba94ffaf231b3 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/717ab9845bb6403c9f7ba94ffaf231b3 2024-11-26T20:42:26,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/717ab9845bb6403c9f7ba94ffaf231b3, entries=150, sequenceid=204, filesize=11.9 K 2024-11-26T20:42:26,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/4433e6e758c34c56a25359a05cf585cb as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/4433e6e758c34c56a25359a05cf585cb 2024-11-26T20:42:26,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/4433e6e758c34c56a25359a05cf585cb, entries=150, sequenceid=204, filesize=11.9 K 2024-11-26T20:42:26,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1591a955d2fc691174b77e9b19a3d332 in 1662ms, sequenceid=204, compaction requested=true 2024-11-26T20:42:26,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:26,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1591a955d2fc691174b77e9b19a3d332:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:42:26,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:26,053 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-26T20:42:26,053 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-26T20:42:26,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1591a955d2fc691174b77e9b19a3d332:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:42:26,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:26,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1591a955d2fc691174b77e9b19a3d332:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:42:26,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:26,054 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 58669 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-26T20:42:26,054 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 164477 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-26T20:42:26,054 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 1591a955d2fc691174b77e9b19a3d332/B is initiating minor compaction (all files) 2024-11-26T20:42:26,054 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 1591a955d2fc691174b77e9b19a3d332/A is initiating minor compaction (all files) 2024-11-26T20:42:26,054 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1591a955d2fc691174b77e9b19a3d332/A in TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:26,054 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1591a955d2fc691174b77e9b19a3d332/B in TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:26,055 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/78ca5409b1ac46dda9db2249b243fc62, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/3644c498a1044bb294494362661c95d6, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/8df528c8e59b4ecf8bd541f20d48934f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/811a914ce45b49dc8f05d2e8d4d0f686, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/717ab9845bb6403c9f7ba94ffaf231b3] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp, totalSize=57.3 K 2024-11-26T20:42:26,055 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/000e32a8b573413db27e32e338f3da78, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ef1100f3aa10427c9b06cb8470c833c4, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/4c3ad95fb7544cc389cfe6581d50b4c9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/135d141f7b7746688eacc56bd80c212d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/c5c154b993b44d889e1182f1738ca4c5] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp, totalSize=160.6 K 2024-11-26T20:42:26,055 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:26,055 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. files: [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/000e32a8b573413db27e32e338f3da78, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ef1100f3aa10427c9b06cb8470c833c4, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/4c3ad95fb7544cc389cfe6581d50b4c9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/135d141f7b7746688eacc56bd80c212d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/c5c154b993b44d889e1182f1738ca4c5] 2024-11-26T20:42:26,055 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 78ca5409b1ac46dda9db2249b243fc62, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732653738690 2024-11-26T20:42:26,055 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 000e32a8b573413db27e32e338f3da78, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732653738690 2024-11-26T20:42:26,055 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 3644c498a1044bb294494362661c95d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732653739382 2024-11-26T20:42:26,056 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef1100f3aa10427c9b06cb8470c833c4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732653739382 2024-11-26T20:42:26,056 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 8df528c8e59b4ecf8bd541f20d48934f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732653740562 2024-11-26T20:42:26,056 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c3ad95fb7544cc389cfe6581d50b4c9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732653740562 2024-11-26T20:42:26,056 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 811a914ce45b49dc8f05d2e8d4d0f686, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732653741809 2024-11-26T20:42:26,056 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 135d141f7b7746688eacc56bd80c212d, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732653741809 2024-11-26T20:42:26,056 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 717ab9845bb6403c9f7ba94ffaf231b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732653744040 2024-11-26T20:42:26,057 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5c154b993b44d889e1182f1738ca4c5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732653744040 2024-11-26T20:42:26,072 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:26,075 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1591a955d2fc691174b77e9b19a3d332#B#compaction#339 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:26,075 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112634ee0e604a1c4474b1db917b0431f19b_1591a955d2fc691174b77e9b19a3d332 store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:26,075 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/03c98461db9a4e6e827df83c6cb707e8 is 50, key is test_row_0/B:col10/1732653744042/Put/seqid=0 2024-11-26T20:42:26,078 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112634ee0e604a1c4474b1db917b0431f19b_1591a955d2fc691174b77e9b19a3d332, store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:26,078 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112634ee0e604a1c4474b1db917b0431f19b_1591a955d2fc691174b77e9b19a3d332 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:26,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742222_1398 (size=12629) 2024-11-26T20:42:26,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742223_1399 (size=4469) 2024-11-26T20:42:26,086 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/03c98461db9a4e6e827df83c6cb707e8 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/03c98461db9a4e6e827df83c6cb707e8 2024-11-26T20:42:26,090 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 1591a955d2fc691174b77e9b19a3d332/B of 1591a955d2fc691174b77e9b19a3d332 into 03c98461db9a4e6e827df83c6cb707e8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:26,090 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:26,090 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., storeName=1591a955d2fc691174b77e9b19a3d332/B, priority=11, startTime=1732653746053; duration=0sec 2024-11-26T20:42:26,090 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:26,090 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1591a955d2fc691174b77e9b19a3d332:B 2024-11-26T20:42:26,090 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-26T20:42:26,092 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 58669 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-26T20:42:26,092 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 1591a955d2fc691174b77e9b19a3d332/C is initiating minor compaction (all files) 2024-11-26T20:42:26,092 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1591a955d2fc691174b77e9b19a3d332/C in TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:26,092 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/9c70fd0c872748f0ac75fe389735bb53, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/c649bf8a41d3472d8d181d11a3ff701d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/2a8597f6e5534f1c972e29021bc07f63, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/a88070697f764ac2a46929c2ff88e4f8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/4433e6e758c34c56a25359a05cf585cb] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp, totalSize=57.3 K 2024-11-26T20:42:26,093 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c70fd0c872748f0ac75fe389735bb53, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732653738690 2024-11-26T20:42:26,093 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting c649bf8a41d3472d8d181d11a3ff701d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732653739382 2024-11-26T20:42:26,094 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a8597f6e5534f1c972e29021bc07f63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732653740562 2024-11-26T20:42:26,094 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting a88070697f764ac2a46929c2ff88e4f8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732653741809 2024-11-26T20:42:26,094 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 4433e6e758c34c56a25359a05cf585cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732653744040 2024-11-26T20:42:26,107 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1591a955d2fc691174b77e9b19a3d332#C#compaction#341 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:26,107 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/ec32808c323d41a9b58186b8728af937 is 50, key is test_row_0/C:col10/1732653744042/Put/seqid=0 2024-11-26T20:42:26,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742224_1400 (size=12629) 2024-11-26T20:42:26,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T20:42:26,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:26,144 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 1591a955d2fc691174b77e9b19a3d332 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T20:42:26,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=A 2024-11-26T20:42:26,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:26,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=B 2024-11-26T20:42:26,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:26,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=C 2024-11-26T20:42:26,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:26,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126603a154d18ab4cc585de4399ab33dd09_1591a955d2fc691174b77e9b19a3d332 is 50, key is test_row_0/A:col10/1732653744442/Put/seqid=0 2024-11-26T20:42:26,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742225_1401 (size=12304) 2024-11-26T20:42:26,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:26,158 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126603a154d18ab4cc585de4399ab33dd09_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126603a154d18ab4cc585de4399ab33dd09_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:26,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/16380134d8d246e488812e1464f75947, store: [table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:26,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/16380134d8d246e488812e1464f75947 is 175, key is test_row_0/A:col10/1732653744442/Put/seqid=0 2024-11-26T20:42:26,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742226_1402 (size=31105) 2024-11-26T20:42:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T20:42:26,483 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1591a955d2fc691174b77e9b19a3d332#A#compaction#340 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:26,484 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/d0703deb840c4cf892536d31d2c118c9 is 175, key is test_row_0/A:col10/1732653744042/Put/seqid=0 2024-11-26T20:42:26,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742227_1403 (size=31583) 2024-11-26T20:42:26,515 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/ec32808c323d41a9b58186b8728af937 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/ec32808c323d41a9b58186b8728af937 2024-11-26T20:42:26,519 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 1591a955d2fc691174b77e9b19a3d332/C of 1591a955d2fc691174b77e9b19a3d332 into ec32808c323d41a9b58186b8728af937(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:26,519 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:26,519 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., storeName=1591a955d2fc691174b77e9b19a3d332/C, priority=11, startTime=1732653746053; duration=0sec 2024-11-26T20:42:26,519 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:26,519 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1591a955d2fc691174b77e9b19a3d332:C 2024-11-26T20:42:26,564 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/16380134d8d246e488812e1464f75947 2024-11-26T20:42:26,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/18ea082cdf324d548ae6568fe95d048f is 50, key is test_row_0/B:col10/1732653744442/Put/seqid=0 2024-11-26T20:42:26,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742228_1404 (size=12151) 2024-11-26T20:42:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:26,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:26,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653806652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:26,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653806653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:26,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653806657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:26,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:26,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653806658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:26,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:26,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653806773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:26,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653806773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:26,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653806773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:26,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653806773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:26,892 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/d0703deb840c4cf892536d31d2c118c9 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/d0703deb840c4cf892536d31d2c118c9 2024-11-26T20:42:26,896 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 1591a955d2fc691174b77e9b19a3d332/A of 1591a955d2fc691174b77e9b19a3d332 into d0703deb840c4cf892536d31d2c118c9(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:26,896 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:26,896 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., storeName=1591a955d2fc691174b77e9b19a3d332/A, priority=11, startTime=1732653746053; duration=0sec 2024-11-26T20:42:26,896 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:26,896 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1591a955d2fc691174b77e9b19a3d332:A 2024-11-26T20:42:26,974 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/18ea082cdf324d548ae6568fe95d048f 2024-11-26T20:42:26,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/2c32f4fcba1d4088b393be408d4aa21e is 50, key is test_row_0/C:col10/1732653744442/Put/seqid=0 2024-11-26T20:42:26,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742229_1405 (size=12151) 2024-11-26T20:42:26,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653806994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:27,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:26,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:27,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653806994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:27,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:27,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653806994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:27,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:27,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653806995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:27,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653807304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:27,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653807304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:27,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:27,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:27,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653807304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:27,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:27,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:27,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653807313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:27,387 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/2c32f4fcba1d4088b393be408d4aa21e 2024-11-26T20:42:27,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/16380134d8d246e488812e1464f75947 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/16380134d8d246e488812e1464f75947 2024-11-26T20:42:27,394 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/16380134d8d246e488812e1464f75947, entries=150, sequenceid=224, filesize=30.4 K 2024-11-26T20:42:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/18ea082cdf324d548ae6568fe95d048f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/18ea082cdf324d548ae6568fe95d048f 2024-11-26T20:42:27,398 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/18ea082cdf324d548ae6568fe95d048f, entries=150, sequenceid=224, filesize=11.9 K 2024-11-26T20:42:27,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/2c32f4fcba1d4088b393be408d4aa21e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/2c32f4fcba1d4088b393be408d4aa21e 2024-11-26T20:42:27,403 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/2c32f4fcba1d4088b393be408d4aa21e, entries=150, sequenceid=224, filesize=11.9 K 2024-11-26T20:42:27,403 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1591a955d2fc691174b77e9b19a3d332 in 1259ms, sequenceid=224, compaction requested=false 2024-11-26T20:42:27,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:27,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:27,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-26T20:42:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-26T20:42:27,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-26T20:42:27,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0980 sec 2024-11-26T20:42:27,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 3.1020 sec 2024-11-26T20:42:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:27,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1591a955d2fc691174b77e9b19a3d332 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T20:42:27,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=A 2024-11-26T20:42:27,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:27,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=B 2024-11-26T20:42:27,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:27,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=C 2024-11-26T20:42:27,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:27,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265a6cb512c3714f9e94cef2f5d6000998_1591a955d2fc691174b77e9b19a3d332 is 50, key is test_row_0/A:col10/1732653746631/Put/seqid=0 2024-11-26T20:42:27,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742230_1406 (size=14794) 2024-11-26T20:42:27,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653807857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:27,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:27,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653807858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:27,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:27,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:27,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653807862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:27,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:27,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:27,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653807863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:27,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:27,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653807984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:27,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:27,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653807984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:27,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:27,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:27,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653807984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:27,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:27,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:27,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653807984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:27,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:27,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:27,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:28,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:28,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653808194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:28,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653808204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:28,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653808204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:28,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:28,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:28,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:28,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:28,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:28,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653808204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:28,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:28,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:28,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:28,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:28,232 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:28,236 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265a6cb512c3714f9e94cef2f5d6000998_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265a6cb512c3714f9e94cef2f5d6000998_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:28,236 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/ac92dbe383cf4454b531b6658ad8e171, store: [table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:28,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/ac92dbe383cf4454b531b6658ad8e171 is 175, key is test_row_0/A:col10/1732653746631/Put/seqid=0 2024-11-26T20:42:28,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742231_1407 (size=39749) 2024-11-26T20:42:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T20:42:28,474 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:42:28,475 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:42:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-26T20:42:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T20:42:28,477 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:42:28,477 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:42:28,478 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:42:28,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653808504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:28,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:28,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:28,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:28,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653808524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:28,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653808524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:28,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:28,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:28,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653808524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:28,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:28,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:28,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:28,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T20:42:28,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-26T20:42:28,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:28,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:28,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:28,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:28,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:28,641 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/ac92dbe383cf4454b531b6658ad8e171 2024-11-26T20:42:28,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/8a956a4799c1415abe4636d4a01832ce is 50, key is test_row_0/B:col10/1732653746631/Put/seqid=0 2024-11-26T20:42:28,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742232_1408 (size=12151) 2024-11-26T20:42:28,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/8a956a4799c1415abe4636d4a01832ce 2024-11-26T20:42:28,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/44e4edaff222462a8d033d8d29b007be is 50, key is test_row_0/C:col10/1732653746631/Put/seqid=0 2024-11-26T20:42:28,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742233_1409 (size=12151) 2024-11-26T20:42:28,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/44e4edaff222462a8d033d8d29b007be 2024-11-26T20:42:28,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/ac92dbe383cf4454b531b6658ad8e171 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ac92dbe383cf4454b531b6658ad8e171 2024-11-26T20:42:28,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ac92dbe383cf4454b531b6658ad8e171, entries=200, sequenceid=240, filesize=38.8 K 2024-11-26T20:42:28,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/8a956a4799c1415abe4636d4a01832ce as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/8a956a4799c1415abe4636d4a01832ce 2024-11-26T20:42:28,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/8a956a4799c1415abe4636d4a01832ce, entries=150, sequenceid=240, filesize=11.9 K 2024-11-26T20:42:28,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/44e4edaff222462a8d033d8d29b007be as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/44e4edaff222462a8d033d8d29b007be 2024-11-26T20:42:28,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/44e4edaff222462a8d033d8d29b007be, entries=150, sequenceid=240, filesize=11.9 K 2024-11-26T20:42:28,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1591a955d2fc691174b77e9b19a3d332 in 867ms, sequenceid=240, compaction requested=true 2024-11-26T20:42:28,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:28,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1591a955d2fc691174b77e9b19a3d332:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:42:28,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:28,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1591a955d2fc691174b77e9b19a3d332:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:42:28,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:28,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1591a955d2fc691174b77e9b19a3d332:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:42:28,686 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:28,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:28,686 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:28,686 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:28,686 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 1591a955d2fc691174b77e9b19a3d332/A is initiating minor compaction (all files) 2024-11-26T20:42:28,687 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1591a955d2fc691174b77e9b19a3d332/A in TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:28,687 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/d0703deb840c4cf892536d31d2c118c9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/16380134d8d246e488812e1464f75947, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ac92dbe383cf4454b531b6658ad8e171] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp, totalSize=100.0 K 2024-11-26T20:42:28,687 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:28,687 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. files: [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/d0703deb840c4cf892536d31d2c118c9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/16380134d8d246e488812e1464f75947, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ac92dbe383cf4454b531b6658ad8e171] 2024-11-26T20:42:28,687 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:28,687 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 1591a955d2fc691174b77e9b19a3d332/B is initiating minor compaction (all files) 2024-11-26T20:42:28,687 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1591a955d2fc691174b77e9b19a3d332/B in TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:28,687 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/03c98461db9a4e6e827df83c6cb707e8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/18ea082cdf324d548ae6568fe95d048f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/8a956a4799c1415abe4636d4a01832ce] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp, totalSize=36.1 K 2024-11-26T20:42:28,687 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0703deb840c4cf892536d31d2c118c9, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732653744040 2024-11-26T20:42:28,688 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 03c98461db9a4e6e827df83c6cb707e8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732653744040 2024-11-26T20:42:28,688 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16380134d8d246e488812e1464f75947, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732653744418 2024-11-26T20:42:28,688 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 18ea082cdf324d548ae6568fe95d048f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732653744418 2024-11-26T20:42:28,688 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac92dbe383cf4454b531b6658ad8e171, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732653746631 2024-11-26T20:42:28,688 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a956a4799c1415abe4636d4a01832ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732653746631 2024-11-26T20:42:28,695 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:28,696 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1591a955d2fc691174b77e9b19a3d332#B#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:28,697 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126c78eb47d8d8d45a687bd9c11a3e87e82_1591a955d2fc691174b77e9b19a3d332 store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:28,697 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/e08c8be4511b431f8f6569f3d44b7721 is 50, key is test_row_0/B:col10/1732653746631/Put/seqid=0 2024-11-26T20:42:28,698 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126c78eb47d8d8d45a687bd9c11a3e87e82_1591a955d2fc691174b77e9b19a3d332, store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:28,698 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c78eb47d8d8d45a687bd9c11a3e87e82_1591a955d2fc691174b77e9b19a3d332 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:28,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742234_1410 (size=4469) 2024-11-26T20:42:28,711 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1591a955d2fc691174b77e9b19a3d332#A#compaction#349 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:28,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742235_1411 (size=12731) 2024-11-26T20:42:28,711 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/ca3900cb09a0448789ca54bd442d109c is 175, key is test_row_0/A:col10/1732653746631/Put/seqid=0 2024-11-26T20:42:28,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742236_1412 (size=31685) 2024-11-26T20:42:28,716 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/e08c8be4511b431f8f6569f3d44b7721 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/e08c8be4511b431f8f6569f3d44b7721 2024-11-26T20:42:28,721 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/ca3900cb09a0448789ca54bd442d109c as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ca3900cb09a0448789ca54bd442d109c 2024-11-26T20:42:28,726 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1591a955d2fc691174b77e9b19a3d332/B of 1591a955d2fc691174b77e9b19a3d332 into e08c8be4511b431f8f6569f3d44b7721(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:28,726 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:28,726 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., storeName=1591a955d2fc691174b77e9b19a3d332/B, priority=13, startTime=1732653748686; duration=0sec 2024-11-26T20:42:28,726 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:28,726 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1591a955d2fc691174b77e9b19a3d332:B 2024-11-26T20:42:28,726 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:28,727 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1591a955d2fc691174b77e9b19a3d332/A of 1591a955d2fc691174b77e9b19a3d332 into ca3900cb09a0448789ca54bd442d109c(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:28,727 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:28,727 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., storeName=1591a955d2fc691174b77e9b19a3d332/A, priority=13, startTime=1732653748686; duration=0sec 2024-11-26T20:42:28,727 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:28,727 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1591a955d2fc691174b77e9b19a3d332:A 2024-11-26T20:42:28,728 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:28,728 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 1591a955d2fc691174b77e9b19a3d332/C is initiating minor compaction (all files) 2024-11-26T20:42:28,728 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1591a955d2fc691174b77e9b19a3d332/C in TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:28,728 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/ec32808c323d41a9b58186b8728af937, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/2c32f4fcba1d4088b393be408d4aa21e, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/44e4edaff222462a8d033d8d29b007be] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp, totalSize=36.1 K 2024-11-26T20:42:28,729 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting ec32808c323d41a9b58186b8728af937, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732653744040 2024-11-26T20:42:28,729 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c32f4fcba1d4088b393be408d4aa21e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732653744418 2024-11-26T20:42:28,729 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 44e4edaff222462a8d033d8d29b007be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732653746631 2024-11-26T20:42:28,736 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1591a955d2fc691174b77e9b19a3d332#C#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:28,737 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/dd56417560114090ab6edc10e1d8a335 is 50, key is test_row_0/C:col10/1732653746631/Put/seqid=0 2024-11-26T20:42:28,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742237_1413 (size=12731) 2024-11-26T20:42:28,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-26T20:42:28,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:28,783 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 1591a955d2fc691174b77e9b19a3d332 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T20:42:28,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=A 2024-11-26T20:42:28,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:28,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=B 2024-11-26T20:42:28,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:28,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=C 2024-11-26T20:42:28,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:28,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112654fc9b61af6c46cd8a80c04bfa7e1190_1591a955d2fc691174b77e9b19a3d332 is 50, key is test_row_0/A:col10/1732653747862/Put/seqid=0 2024-11-26T20:42:28,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742238_1414 (size=12354) 2024-11-26T20:42:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T20:42:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:29,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. as already flushing 2024-11-26T20:42:29,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653809059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:29,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:29,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653809067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653809068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:29,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653809068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T20:42:29,144 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/dd56417560114090ab6edc10e1d8a335 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/dd56417560114090ab6edc10e1d8a335 2024-11-26T20:42:29,148 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1591a955d2fc691174b77e9b19a3d332/C of 1591a955d2fc691174b77e9b19a3d332 into dd56417560114090ab6edc10e1d8a335(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:29,148 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:29,148 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., storeName=1591a955d2fc691174b77e9b19a3d332/C, priority=13, startTime=1732653748686; duration=0sec 2024-11-26T20:42:29,148 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:29,148 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1591a955d2fc691174b77e9b19a3d332:C 2024-11-26T20:42:29,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653809174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:29,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:29,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:29,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653809174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653809174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:29,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:29,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:29,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653809184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:29,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:29,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:29,195 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112654fc9b61af6c46cd8a80c04bfa7e1190_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112654fc9b61af6c46cd8a80c04bfa7e1190_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:29,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/2a11598ee3bd4abaa0f55bfaadb3dee6, store: [table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:29,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/2a11598ee3bd4abaa0f55bfaadb3dee6 is 175, key is test_row_0/A:col10/1732653747862/Put/seqid=0 2024-11-26T20:42:29,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742239_1415 (size=31155) 2024-11-26T20:42:29,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653809394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:29,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653809394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:29,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653809394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:29,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653809394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:29,601 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/2a11598ee3bd4abaa0f55bfaadb3dee6 2024-11-26T20:42:29,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/708824cc99674631bbc4238e882d1b27 is 50, key is test_row_0/B:col10/1732653747862/Put/seqid=0 2024-11-26T20:42:29,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742240_1416 (size=12201) 2024-11-26T20:42:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T20:42:29,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653809713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:29,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:29,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653809714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:29,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:29,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653809714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:29,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:29,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:29,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653809714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:29,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:29,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:29,967 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T20:42:30,011 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/708824cc99674631bbc4238e882d1b27 2024-11-26T20:42:30,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/8d1f654c66a34b8a92000f6a46a15faf is 50, key is test_row_0/C:col10/1732653747862/Put/seqid=0 2024-11-26T20:42:30,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742241_1417 (size=12201) 2024-11-26T20:42:30,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50592 deadline: 1732653810224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:30,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50524 deadline: 1732653810224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:30,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:30,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50580 deadline: 1732653810224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:30,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:30,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:30,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50578 deadline: 1732653810233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:42:30,420 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/8d1f654c66a34b8a92000f6a46a15faf 2024-11-26T20:42:30,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/2a11598ee3bd4abaa0f55bfaadb3dee6 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/2a11598ee3bd4abaa0f55bfaadb3dee6 2024-11-26T20:42:30,427 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/2a11598ee3bd4abaa0f55bfaadb3dee6, entries=150, sequenceid=260, filesize=30.4 K 2024-11-26T20:42:30,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/708824cc99674631bbc4238e882d1b27 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/708824cc99674631bbc4238e882d1b27 2024-11-26T20:42:30,431 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/708824cc99674631bbc4238e882d1b27, entries=150, sequenceid=260, filesize=11.9 K 2024-11-26T20:42:30,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/8d1f654c66a34b8a92000f6a46a15faf as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/8d1f654c66a34b8a92000f6a46a15faf 2024-11-26T20:42:30,436 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/8d1f654c66a34b8a92000f6a46a15faf, entries=150, sequenceid=260, filesize=11.9 K 2024-11-26T20:42:30,436 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1591a955d2fc691174b77e9b19a3d332 in 1653ms, sequenceid=260, compaction requested=false 2024-11-26T20:42:30,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:30,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:30,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-26T20:42:30,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-26T20:42:30,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-26T20:42:30,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9610 sec 2024-11-26T20:42:30,441 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.9650 sec 2024-11-26T20:42:30,502 INFO [Thread-1573 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1573. 2024-11-26T20:42:30,503 DEBUG [Thread-1573 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:42:30,503 DEBUG [Thread-1573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:30,503 DEBUG [Thread-1573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:30,503 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:42:30,504 INFO [Thread-1571 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1571. 2024-11-26T20:42:30,504 DEBUG [Thread-1571 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:42:30,504 DEBUG [Thread-1571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:30,504 DEBUG [Thread-1571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:30,505 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:42:30,508 INFO [Thread-1565 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1565. 2024-11-26T20:42:30,508 DEBUG [Thread-1565 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:42:30,508 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:30,508 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:30,508 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:42:30,509 INFO [Thread-1567 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1567. 2024-11-26T20:42:30,509 DEBUG [Thread-1567 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:42:30,509 DEBUG [Thread-1567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:30,509 DEBUG [Thread-1567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:30,509 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:42:30,511 INFO [Thread-1569 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1569. 2024-11-26T20:42:30,511 DEBUG [Thread-1569 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:42:30,511 DEBUG [Thread-1569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:30,511 DEBUG [Thread-1569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:30,511 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:42:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T20:42:30,624 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:42:31,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:31,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1591a955d2fc691174b77e9b19a3d332 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T20:42:31,245 INFO [Thread-1558 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1558. 2024-11-26T20:42:31,245 INFO [Thread-1562 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1562. 2024-11-26T20:42:31,245 DEBUG [Thread-1562 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:42:31,245 DEBUG [Thread-1558 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:42:31,245 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:31,245 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:31,245 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:31,245 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:31,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=A 2024-11-26T20:42:31,245 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:42:31,245 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:42:31,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:31,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=B 2024-11-26T20:42:31,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:31,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=C 2024-11-26T20:42:31,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:31,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126772e4e9efb8f4ad8a6bd60c930d7a903_1591a955d2fc691174b77e9b19a3d332 is 50, key is test_row_0/A:col10/1732653751244/Put/seqid=0 2024-11-26T20:42:31,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742242_1418 (size=12454) 2024-11-26T20:42:31,255 INFO [Thread-1554 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1554. 2024-11-26T20:42:31,255 DEBUG [Thread-1554 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:42:31,255 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:31,255 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:31,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:42:31,255 INFO [Thread-1556 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1556. 2024-11-26T20:42:31,255 DEBUG [Thread-1556 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:42:31,255 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:31,255 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:31,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:42:31,654 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:31,656 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126772e4e9efb8f4ad8a6bd60c930d7a903_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126772e4e9efb8f4ad8a6bd60c930d7a903_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:31,657 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/5682a945c161407ab4ce88c37f5ef06b, store: [table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:31,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/5682a945c161407ab4ce88c37f5ef06b is 175, key is test_row_0/A:col10/1732653751244/Put/seqid=0 2024-11-26T20:42:31,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742243_1419 (size=31255) 2024-11-26T20:42:32,061 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/5682a945c161407ab4ce88c37f5ef06b 2024-11-26T20:42:32,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/7d6cdc43f66c429992303f1592338045 is 50, key is test_row_0/B:col10/1732653751244/Put/seqid=0 2024-11-26T20:42:32,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742244_1420 (size=12301) 2024-11-26T20:42:32,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/7d6cdc43f66c429992303f1592338045 2024-11-26T20:42:32,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/def1fed2b707481db09be98f7d966d99 is 50, key is test_row_0/C:col10/1732653751244/Put/seqid=0 2024-11-26T20:42:32,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742245_1421 (size=12301) 2024-11-26T20:42:32,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/def1fed2b707481db09be98f7d966d99 2024-11-26T20:42:32,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/5682a945c161407ab4ce88c37f5ef06b as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/5682a945c161407ab4ce88c37f5ef06b 2024-11-26T20:42:32,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/5682a945c161407ab4ce88c37f5ef06b, entries=150, sequenceid=277, filesize=30.5 K 2024-11-26T20:42:32,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/7d6cdc43f66c429992303f1592338045 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/7d6cdc43f66c429992303f1592338045 2024-11-26T20:42:32,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/7d6cdc43f66c429992303f1592338045, entries=150, sequenceid=277, filesize=12.0 K 2024-11-26T20:42:32,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/def1fed2b707481db09be98f7d966d99 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/def1fed2b707481db09be98f7d966d99 2024-11-26T20:42:32,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/def1fed2b707481db09be98f7d966d99, entries=150, sequenceid=277, filesize=12.0 K 2024-11-26T20:42:32,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=13.42 KB/13740 for 1591a955d2fc691174b77e9b19a3d332 in 1649ms, sequenceid=277, compaction requested=true 2024-11-26T20:42:32,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:32,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1591a955d2fc691174b77e9b19a3d332:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:42:32,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:32,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1591a955d2fc691174b77e9b19a3d332:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:42:32,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:32,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1591a955d2fc691174b77e9b19a3d332:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:42:32,894 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:32,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:32,894 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:32,895 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:32,895 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:32,895 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 1591a955d2fc691174b77e9b19a3d332/B is initiating minor compaction (all files) 2024-11-26T20:42:32,895 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 1591a955d2fc691174b77e9b19a3d332/A is initiating minor compaction (all files) 2024-11-26T20:42:32,895 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1591a955d2fc691174b77e9b19a3d332/A in TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:32,895 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1591a955d2fc691174b77e9b19a3d332/B in TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:32,895 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ca3900cb09a0448789ca54bd442d109c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/2a11598ee3bd4abaa0f55bfaadb3dee6, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/5682a945c161407ab4ce88c37f5ef06b] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp, totalSize=91.9 K 2024-11-26T20:42:32,895 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/e08c8be4511b431f8f6569f3d44b7721, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/708824cc99674631bbc4238e882d1b27, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/7d6cdc43f66c429992303f1592338045] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp, totalSize=36.4 K 2024-11-26T20:42:32,895 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:32,895 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. files: [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ca3900cb09a0448789ca54bd442d109c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/2a11598ee3bd4abaa0f55bfaadb3dee6, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/5682a945c161407ab4ce88c37f5ef06b] 2024-11-26T20:42:32,896 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting e08c8be4511b431f8f6569f3d44b7721, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732653746631 2024-11-26T20:42:32,896 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca3900cb09a0448789ca54bd442d109c, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732653746631 2024-11-26T20:42:32,896 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 708824cc99674631bbc4238e882d1b27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732653747840 2024-11-26T20:42:32,896 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a11598ee3bd4abaa0f55bfaadb3dee6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732653747840 2024-11-26T20:42:32,896 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d6cdc43f66c429992303f1592338045, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732653749053 2024-11-26T20:42:32,896 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5682a945c161407ab4ce88c37f5ef06b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732653749053 2024-11-26T20:42:32,902 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1591a955d2fc691174b77e9b19a3d332#B#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:32,902 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/45bcb79c619f43afb12ab47893a2ee6e is 50, key is test_row_0/B:col10/1732653751244/Put/seqid=0 2024-11-26T20:42:32,905 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:32,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742246_1422 (size=12983) 2024-11-26T20:42:32,907 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126e8faf54d43cf434f9ba6ebcfdf1115df_1591a955d2fc691174b77e9b19a3d332 store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:32,922 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126e8faf54d43cf434f9ba6ebcfdf1115df_1591a955d2fc691174b77e9b19a3d332, store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:32,922 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e8faf54d43cf434f9ba6ebcfdf1115df_1591a955d2fc691174b77e9b19a3d332 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:32,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742247_1423 (size=4469) 2024-11-26T20:42:33,310 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/45bcb79c619f43afb12ab47893a2ee6e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/45bcb79c619f43afb12ab47893a2ee6e 2024-11-26T20:42:33,313 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1591a955d2fc691174b77e9b19a3d332/B of 1591a955d2fc691174b77e9b19a3d332 into 45bcb79c619f43afb12ab47893a2ee6e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:33,314 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:33,314 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., storeName=1591a955d2fc691174b77e9b19a3d332/B, priority=13, startTime=1732653752894; duration=0sec 2024-11-26T20:42:33,314 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:33,314 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1591a955d2fc691174b77e9b19a3d332:B 2024-11-26T20:42:33,314 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:33,314 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:33,315 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 1591a955d2fc691174b77e9b19a3d332/C is initiating minor compaction (all files) 2024-11-26T20:42:33,315 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1591a955d2fc691174b77e9b19a3d332/C in TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:33,315 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/dd56417560114090ab6edc10e1d8a335, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/8d1f654c66a34b8a92000f6a46a15faf, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/def1fed2b707481db09be98f7d966d99] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp, totalSize=36.4 K 2024-11-26T20:42:33,315 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting dd56417560114090ab6edc10e1d8a335, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732653746631 2024-11-26T20:42:33,315 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d1f654c66a34b8a92000f6a46a15faf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732653747840 2024-11-26T20:42:33,315 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting def1fed2b707481db09be98f7d966d99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732653749053 2024-11-26T20:42:33,321 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1591a955d2fc691174b77e9b19a3d332#C#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:33,321 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/6fc03c1ffa29406cae3eafc895b7f7d3 is 50, key is test_row_0/C:col10/1732653751244/Put/seqid=0 2024-11-26T20:42:33,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742248_1424 (size=12983) 2024-11-26T20:42:33,327 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1591a955d2fc691174b77e9b19a3d332#A#compaction#358 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:33,328 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/ac8dae82a6cb4959872def36ee44f73c is 175, key is test_row_0/A:col10/1732653751244/Put/seqid=0 2024-11-26T20:42:33,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742249_1425 (size=31937) 2024-11-26T20:42:33,334 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/ac8dae82a6cb4959872def36ee44f73c as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ac8dae82a6cb4959872def36ee44f73c 2024-11-26T20:42:33,337 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1591a955d2fc691174b77e9b19a3d332/A of 1591a955d2fc691174b77e9b19a3d332 into ac8dae82a6cb4959872def36ee44f73c(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:33,337 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:33,337 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., storeName=1591a955d2fc691174b77e9b19a3d332/A, priority=13, startTime=1732653752894; duration=0sec 2024-11-26T20:42:33,337 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:33,337 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1591a955d2fc691174b77e9b19a3d332:A 2024-11-26T20:42:33,729 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/6fc03c1ffa29406cae3eafc895b7f7d3 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/6fc03c1ffa29406cae3eafc895b7f7d3 2024-11-26T20:42:33,732 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1591a955d2fc691174b77e9b19a3d332/C of 1591a955d2fc691174b77e9b19a3d332 into 6fc03c1ffa29406cae3eafc895b7f7d3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:33,732 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1591a955d2fc691174b77e9b19a3d332: 2024-11-26T20:42:33,732 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., storeName=1591a955d2fc691174b77e9b19a3d332/C, priority=13, startTime=1732653752894; duration=0sec 2024-11-26T20:42:33,732 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:33,732 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1591a955d2fc691174b77e9b19a3d332:C 2024-11-26T20:42:33,839 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/125605ae78c840dd9d31ccc3709f58c7, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/495332bcfe99442f8827ba863256ac0a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/de05e878e8b44f228349f9a866329735, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/b8576971b5d34391a564c813339c804e, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/392558849c284788b34627809c7e8934, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/b9e92bf5869541e880f08c91e2d71eb3, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/04d4dffa2f9642c4ba7a145b6b669eb5, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/3e083b7a070d4b78aed72c14caf5b35c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/000e32a8b573413db27e32e338f3da78, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/e500e20efe9a4286931c174cac4acb77, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ef1100f3aa10427c9b06cb8470c833c4, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/4c3ad95fb7544cc389cfe6581d50b4c9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/135d141f7b7746688eacc56bd80c212d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/c5c154b993b44d889e1182f1738ca4c5, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/d0703deb840c4cf892536d31d2c118c9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/16380134d8d246e488812e1464f75947, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ac92dbe383cf4454b531b6658ad8e171, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ca3900cb09a0448789ca54bd442d109c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/2a11598ee3bd4abaa0f55bfaadb3dee6, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/5682a945c161407ab4ce88c37f5ef06b] to archive 2024-11-26T20:42:33,839 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T20:42:33,841 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/125605ae78c840dd9d31ccc3709f58c7 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/125605ae78c840dd9d31ccc3709f58c7 2024-11-26T20:42:33,841 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/495332bcfe99442f8827ba863256ac0a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/495332bcfe99442f8827ba863256ac0a 2024-11-26T20:42:33,842 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/de05e878e8b44f228349f9a866329735 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/de05e878e8b44f228349f9a866329735 2024-11-26T20:42:33,843 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/b8576971b5d34391a564c813339c804e to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/b8576971b5d34391a564c813339c804e 2024-11-26T20:42:33,844 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/392558849c284788b34627809c7e8934 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/392558849c284788b34627809c7e8934 2024-11-26T20:42:33,845 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/b9e92bf5869541e880f08c91e2d71eb3 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/b9e92bf5869541e880f08c91e2d71eb3 2024-11-26T20:42:33,845 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/04d4dffa2f9642c4ba7a145b6b669eb5 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/04d4dffa2f9642c4ba7a145b6b669eb5 2024-11-26T20:42:33,846 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/3e083b7a070d4b78aed72c14caf5b35c to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/3e083b7a070d4b78aed72c14caf5b35c 2024-11-26T20:42:33,847 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/000e32a8b573413db27e32e338f3da78 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/000e32a8b573413db27e32e338f3da78 2024-11-26T20:42:33,848 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/e500e20efe9a4286931c174cac4acb77 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/e500e20efe9a4286931c174cac4acb77 2024-11-26T20:42:33,848 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ef1100f3aa10427c9b06cb8470c833c4 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ef1100f3aa10427c9b06cb8470c833c4 2024-11-26T20:42:33,849 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/4c3ad95fb7544cc389cfe6581d50b4c9 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/4c3ad95fb7544cc389cfe6581d50b4c9 2024-11-26T20:42:33,850 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/135d141f7b7746688eacc56bd80c212d to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/135d141f7b7746688eacc56bd80c212d 2024-11-26T20:42:33,851 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/c5c154b993b44d889e1182f1738ca4c5 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/c5c154b993b44d889e1182f1738ca4c5 2024-11-26T20:42:33,852 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/d0703deb840c4cf892536d31d2c118c9 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/d0703deb840c4cf892536d31d2c118c9 2024-11-26T20:42:33,852 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/16380134d8d246e488812e1464f75947 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/16380134d8d246e488812e1464f75947 2024-11-26T20:42:33,853 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ac92dbe383cf4454b531b6658ad8e171 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ac92dbe383cf4454b531b6658ad8e171 2024-11-26T20:42:33,854 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ca3900cb09a0448789ca54bd442d109c to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ca3900cb09a0448789ca54bd442d109c 2024-11-26T20:42:33,855 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/2a11598ee3bd4abaa0f55bfaadb3dee6 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/2a11598ee3bd4abaa0f55bfaadb3dee6 2024-11-26T20:42:33,856 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/5682a945c161407ab4ce88c37f5ef06b to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/5682a945c161407ab4ce88c37f5ef06b 2024-11-26T20:42:33,858 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/3e931976a35441af8f75a20added0ecb, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/0e73a1650e214a918a8129acf203fd59, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/8b04144eccf7450f8ed4c36c78034496, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/767202d0f4ab42b89a89a9be7ca5ea78, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/9d461decebb74bc09052708739541fb8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/b579b8b978224134b7951bc6a2b2f6b4, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/905c372b364f4ea4ace26bcb80971470, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/6827d9da919648e588dfe38d25707c04, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/78ca5409b1ac46dda9db2249b243fc62, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/442daf682217497ca81b5b3a447b4457, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/3644c498a1044bb294494362661c95d6, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/8df528c8e59b4ecf8bd541f20d48934f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/811a914ce45b49dc8f05d2e8d4d0f686, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/03c98461db9a4e6e827df83c6cb707e8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/717ab9845bb6403c9f7ba94ffaf231b3, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/18ea082cdf324d548ae6568fe95d048f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/e08c8be4511b431f8f6569f3d44b7721, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/8a956a4799c1415abe4636d4a01832ce, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/708824cc99674631bbc4238e882d1b27, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/7d6cdc43f66c429992303f1592338045] to archive 2024-11-26T20:42:33,859 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T20:42:33,860 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/3e931976a35441af8f75a20added0ecb to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/3e931976a35441af8f75a20added0ecb 2024-11-26T20:42:33,861 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/0e73a1650e214a918a8129acf203fd59 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/0e73a1650e214a918a8129acf203fd59 2024-11-26T20:42:33,862 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/8b04144eccf7450f8ed4c36c78034496 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/8b04144eccf7450f8ed4c36c78034496 2024-11-26T20:42:33,862 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/767202d0f4ab42b89a89a9be7ca5ea78 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/767202d0f4ab42b89a89a9be7ca5ea78 2024-11-26T20:42:33,863 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/9d461decebb74bc09052708739541fb8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/9d461decebb74bc09052708739541fb8 2024-11-26T20:42:33,864 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/b579b8b978224134b7951bc6a2b2f6b4 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/b579b8b978224134b7951bc6a2b2f6b4 2024-11-26T20:42:33,865 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/905c372b364f4ea4ace26bcb80971470 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/905c372b364f4ea4ace26bcb80971470 2024-11-26T20:42:33,865 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/6827d9da919648e588dfe38d25707c04 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/6827d9da919648e588dfe38d25707c04 2024-11-26T20:42:33,866 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/78ca5409b1ac46dda9db2249b243fc62 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/78ca5409b1ac46dda9db2249b243fc62 2024-11-26T20:42:33,867 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/442daf682217497ca81b5b3a447b4457 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/442daf682217497ca81b5b3a447b4457 2024-11-26T20:42:33,868 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/3644c498a1044bb294494362661c95d6 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/3644c498a1044bb294494362661c95d6 2024-11-26T20:42:33,868 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/8df528c8e59b4ecf8bd541f20d48934f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/8df528c8e59b4ecf8bd541f20d48934f 2024-11-26T20:42:33,869 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/811a914ce45b49dc8f05d2e8d4d0f686 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/811a914ce45b49dc8f05d2e8d4d0f686 2024-11-26T20:42:33,870 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/03c98461db9a4e6e827df83c6cb707e8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/03c98461db9a4e6e827df83c6cb707e8 2024-11-26T20:42:33,871 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/717ab9845bb6403c9f7ba94ffaf231b3 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/717ab9845bb6403c9f7ba94ffaf231b3 2024-11-26T20:42:33,871 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/18ea082cdf324d548ae6568fe95d048f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/18ea082cdf324d548ae6568fe95d048f 2024-11-26T20:42:33,872 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/e08c8be4511b431f8f6569f3d44b7721 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/e08c8be4511b431f8f6569f3d44b7721 2024-11-26T20:42:33,873 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/8a956a4799c1415abe4636d4a01832ce to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/8a956a4799c1415abe4636d4a01832ce 2024-11-26T20:42:33,874 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/708824cc99674631bbc4238e882d1b27 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/708824cc99674631bbc4238e882d1b27 2024-11-26T20:42:33,874 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/7d6cdc43f66c429992303f1592338045 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/7d6cdc43f66c429992303f1592338045 2024-11-26T20:42:33,877 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/bf8dcab11afb4a6aa8084b0e0dd74309, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/d8a6d650cb924274be9daee0efbbee25, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/8008a65efd3e44c7a301fa528ac5fe3d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/f89b4efd23d8434f8172d510a5547060, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/b69d81d26d564ea0a401a0a274de50ae, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/3061bdaa8aef49acaa273f235283294a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/91a05d5738864992a24e76cdb298c2a8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/02fed51a18d5483796765a05d612f51f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/9c70fd0c872748f0ac75fe389735bb53, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/9f53df94694848edb313e406892fc029, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/c649bf8a41d3472d8d181d11a3ff701d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/2a8597f6e5534f1c972e29021bc07f63, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/a88070697f764ac2a46929c2ff88e4f8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/ec32808c323d41a9b58186b8728af937, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/4433e6e758c34c56a25359a05cf585cb, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/2c32f4fcba1d4088b393be408d4aa21e, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/dd56417560114090ab6edc10e1d8a335, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/44e4edaff222462a8d033d8d29b007be, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/8d1f654c66a34b8a92000f6a46a15faf, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/def1fed2b707481db09be98f7d966d99] to archive 2024-11-26T20:42:33,877 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T20:42:33,878 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/bf8dcab11afb4a6aa8084b0e0dd74309 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/bf8dcab11afb4a6aa8084b0e0dd74309 2024-11-26T20:42:33,879 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/d8a6d650cb924274be9daee0efbbee25 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/d8a6d650cb924274be9daee0efbbee25 2024-11-26T20:42:33,880 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/8008a65efd3e44c7a301fa528ac5fe3d to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/8008a65efd3e44c7a301fa528ac5fe3d 2024-11-26T20:42:33,881 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/f89b4efd23d8434f8172d510a5547060 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/f89b4efd23d8434f8172d510a5547060 2024-11-26T20:42:33,881 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/b69d81d26d564ea0a401a0a274de50ae to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/b69d81d26d564ea0a401a0a274de50ae 2024-11-26T20:42:33,882 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/3061bdaa8aef49acaa273f235283294a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/3061bdaa8aef49acaa273f235283294a 2024-11-26T20:42:33,883 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/91a05d5738864992a24e76cdb298c2a8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/91a05d5738864992a24e76cdb298c2a8 2024-11-26T20:42:33,884 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/02fed51a18d5483796765a05d612f51f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/02fed51a18d5483796765a05d612f51f 2024-11-26T20:42:33,884 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/9c70fd0c872748f0ac75fe389735bb53 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/9c70fd0c872748f0ac75fe389735bb53 2024-11-26T20:42:33,885 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/9f53df94694848edb313e406892fc029 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/9f53df94694848edb313e406892fc029 2024-11-26T20:42:33,886 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/c649bf8a41d3472d8d181d11a3ff701d to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/c649bf8a41d3472d8d181d11a3ff701d 2024-11-26T20:42:33,887 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/2a8597f6e5534f1c972e29021bc07f63 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/2a8597f6e5534f1c972e29021bc07f63 2024-11-26T20:42:33,887 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/a88070697f764ac2a46929c2ff88e4f8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/a88070697f764ac2a46929c2ff88e4f8 2024-11-26T20:42:33,888 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/ec32808c323d41a9b58186b8728af937 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/ec32808c323d41a9b58186b8728af937 2024-11-26T20:42:33,889 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/4433e6e758c34c56a25359a05cf585cb to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/4433e6e758c34c56a25359a05cf585cb 2024-11-26T20:42:33,890 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/2c32f4fcba1d4088b393be408d4aa21e to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/2c32f4fcba1d4088b393be408d4aa21e 2024-11-26T20:42:33,890 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/dd56417560114090ab6edc10e1d8a335 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/dd56417560114090ab6edc10e1d8a335 2024-11-26T20:42:33,891 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/44e4edaff222462a8d033d8d29b007be to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/44e4edaff222462a8d033d8d29b007be 2024-11-26T20:42:33,892 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/8d1f654c66a34b8a92000f6a46a15faf to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/8d1f654c66a34b8a92000f6a46a15faf 2024-11-26T20:42:33,893 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/def1fed2b707481db09be98f7d966d99 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/def1fed2b707481db09be98f7d966d99 2024-11-26T20:42:34,795 INFO [Thread-1560 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1560. 2024-11-26T20:42:34,796 DEBUG [Thread-1560 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:42:34,796 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:34,796 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:34,796 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:42:34,796 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T20:42:34,796 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-26T20:42:34,796 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-26T20:42:34,796 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-26T20:42:34,796 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-26T20:42:34,796 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-26T20:42:34,796 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T20:42:34,796 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T20:42:34,796 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1080 2024-11-26T20:42:34,796 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3235 rows 2024-11-26T20:42:34,796 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1082 2024-11-26T20:42:34,796 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3241 rows 2024-11-26T20:42:34,796 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1086 2024-11-26T20:42:34,796 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3255 rows 2024-11-26T20:42:34,796 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1074 2024-11-26T20:42:34,796 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3216 rows 2024-11-26T20:42:34,796 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1081 2024-11-26T20:42:34,796 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3238 rows 2024-11-26T20:42:34,796 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T20:42:34,796 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:42:34,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:34,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:34,796 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:42:34,797 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T20:42:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T20:42:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T20:42:34,800 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653754800"}]},"ts":"1732653754800"} 2024-11-26T20:42:34,801 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T20:42:34,801 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T20:42:34,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T20:42:34,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1591a955d2fc691174b77e9b19a3d332, UNASSIGN}] 2024-11-26T20:42:34,803 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1591a955d2fc691174b77e9b19a3d332, UNASSIGN 2024-11-26T20:42:34,804 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=1591a955d2fc691174b77e9b19a3d332, regionState=CLOSING, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:42:34,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1591a955d2fc691174b77e9b19a3d332, UNASSIGN because future has completed 2024-11-26T20:42:34,805 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T20:42:34,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795}] 2024-11-26T20:42:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T20:42:34,957 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(122): Close 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:34,957 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T20:42:34,957 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1722): Closing 1591a955d2fc691174b77e9b19a3d332, disabling compactions & flushes 2024-11-26T20:42:34,957 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:34,957 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:34,957 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. after waiting 0 ms 2024-11-26T20:42:34,957 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:34,957 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(2902): Flushing 1591a955d2fc691174b77e9b19a3d332 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-26T20:42:34,957 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=A 2024-11-26T20:42:34,957 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:34,957 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=B 2024-11-26T20:42:34,957 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:34,957 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1591a955d2fc691174b77e9b19a3d332, store=C 2024-11-26T20:42:34,958 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:34,962 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126840c380cc00d406ca22bdb2b9fad88e2_1591a955d2fc691174b77e9b19a3d332 is 50, key is test_row_0/A:col10/1732653754794/Put/seqid=0 2024-11-26T20:42:34,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742250_1426 (size=7374) 2024-11-26T20:42:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T20:42:35,365 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:35,368 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126840c380cc00d406ca22bdb2b9fad88e2_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126840c380cc00d406ca22bdb2b9fad88e2_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:35,369 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/b22df265c46841269e262e23a17accbb, store: [table=TestAcidGuarantees family=A region=1591a955d2fc691174b77e9b19a3d332] 2024-11-26T20:42:35,369 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/b22df265c46841269e262e23a17accbb is 175, key is test_row_0/A:col10/1732653754794/Put/seqid=0 2024-11-26T20:42:35,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742251_1427 (size=13865) 2024-11-26T20:42:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T20:42:35,773 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/b22df265c46841269e262e23a17accbb 2024-11-26T20:42:35,778 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/2f9e6e38f8284baa93f71821f1de60a5 is 50, key is test_row_0/B:col10/1732653754794/Put/seqid=0 2024-11-26T20:42:35,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742252_1428 (size=7415) 2024-11-26T20:42:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T20:42:36,182 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/2f9e6e38f8284baa93f71821f1de60a5 2024-11-26T20:42:36,187 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/b344a3c4590241b4a7cc6ca6d7b3d6f8 is 50, key is test_row_0/C:col10/1732653754794/Put/seqid=0 2024-11-26T20:42:36,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742253_1429 (size=7415) 2024-11-26T20:42:36,591 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/b344a3c4590241b4a7cc6ca6d7b3d6f8 2024-11-26T20:42:36,595 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/A/b22df265c46841269e262e23a17accbb as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/b22df265c46841269e262e23a17accbb 2024-11-26T20:42:36,597 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/b22df265c46841269e262e23a17accbb, entries=50, sequenceid=286, filesize=13.5 K 2024-11-26T20:42:36,598 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/B/2f9e6e38f8284baa93f71821f1de60a5 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/2f9e6e38f8284baa93f71821f1de60a5 2024-11-26T20:42:36,601 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/2f9e6e38f8284baa93f71821f1de60a5, entries=50, sequenceid=286, filesize=7.2 K 2024-11-26T20:42:36,601 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/.tmp/C/b344a3c4590241b4a7cc6ca6d7b3d6f8 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/b344a3c4590241b4a7cc6ca6d7b3d6f8 2024-11-26T20:42:36,604 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/b344a3c4590241b4a7cc6ca6d7b3d6f8, entries=50, sequenceid=286, filesize=7.2 K 2024-11-26T20:42:36,604 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 1591a955d2fc691174b77e9b19a3d332 in 1647ms, sequenceid=286, compaction requested=false 2024-11-26T20:42:36,608 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/recovered.edits/289.seqid, newMaxSeqId=289, maxSeqId=4 2024-11-26T20:42:36,608 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. 2024-11-26T20:42:36,608 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1676): Region close journal for 1591a955d2fc691174b77e9b19a3d332: Waiting for close lock at 1732653754957Running coprocessor pre-close hooks at 1732653754957Disabling compacts and flushes for region at 1732653754957Disabling writes for close at 1732653754957Obtaining lock to block concurrent updates at 1732653754957Preparing flush snapshotting stores in 1591a955d2fc691174b77e9b19a3d332 at 1732653754957Finished memstore snapshotting TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732653754958 (+1 ms)Flushing stores of TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332. at 1732653754958Flushing 1591a955d2fc691174b77e9b19a3d332/A: creating writer at 1732653754958Flushing mob file 1591a955d2fc691174b77e9b19a3d332/A: appending metadata at 1732653754962 (+4 ms)Flushing mob file 1591a955d2fc691174b77e9b19a3d332/A: closing flushed file at 1732653754962Flushing 1591a955d2fc691174b77e9b19a3d332/A: appending metadata at 1732653755369 (+407 ms)Flushing 1591a955d2fc691174b77e9b19a3d332/A: closing flushed file at 1732653755369Flushing 1591a955d2fc691174b77e9b19a3d332/B: creating writer at 1732653755776 (+407 ms)Flushing 1591a955d2fc691174b77e9b19a3d332/B: appending metadata at 1732653755778 (+2 ms)Flushing 1591a955d2fc691174b77e9b19a3d332/B: closing flushed file at 1732653755778Flushing 1591a955d2fc691174b77e9b19a3d332/C: creating writer at 1732653756185 (+407 ms)Flushing 1591a955d2fc691174b77e9b19a3d332/C: appending metadata at 1732653756187 (+2 ms)Flushing 1591a955d2fc691174b77e9b19a3d332/C: closing flushed file at 1732653756187Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4be041af: reopening flushed file at 1732653756594 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f96f8d5: reopening flushed file at 1732653756598 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2427e9b: reopening flushed file at 1732653756601 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 1591a955d2fc691174b77e9b19a3d332 in 1647ms, sequenceid=286, compaction requested=false at 1732653756604 (+3 ms)Writing region close event to WAL at 1732653756606 (+2 ms)Running coprocessor post-close hooks at 1732653756608 (+2 ms)Closed at 1732653756608 2024-11-26T20:42:36,610 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(157): Closed 1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:36,611 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=1591a955d2fc691174b77e9b19a3d332, regionState=CLOSED 2024-11-26T20:42:36,612 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 because future has completed 2024-11-26T20:42:36,614 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-26T20:42:36,614 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseRegionProcedure 1591a955d2fc691174b77e9b19a3d332, server=cd6be161543e,40291,1732653631795 in 1.8070 sec 2024-11-26T20:42:36,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-26T20:42:36,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1591a955d2fc691174b77e9b19a3d332, UNASSIGN in 1.8110 sec 2024-11-26T20:42:36,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-26T20:42:36,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8140 sec 2024-11-26T20:42:36,618 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653756618"}]},"ts":"1732653756618"} 2024-11-26T20:42:36,619 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T20:42:36,619 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T20:42:36,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8220 sec 2024-11-26T20:42:36,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T20:42:36,944 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:42:36,945 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T20:42:36,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:42:36,946 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:42:36,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T20:42:36,947 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=119, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:42:36,949 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:36,950 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A, FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B, FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C, FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/recovered.edits] 2024-11-26T20:42:36,952 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ac8dae82a6cb4959872def36ee44f73c to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/ac8dae82a6cb4959872def36ee44f73c 2024-11-26T20:42:36,953 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/b22df265c46841269e262e23a17accbb to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/A/b22df265c46841269e262e23a17accbb 2024-11-26T20:42:36,955 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/2f9e6e38f8284baa93f71821f1de60a5 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/2f9e6e38f8284baa93f71821f1de60a5 2024-11-26T20:42:36,956 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/45bcb79c619f43afb12ab47893a2ee6e to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/B/45bcb79c619f43afb12ab47893a2ee6e 2024-11-26T20:42:36,957 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/6fc03c1ffa29406cae3eafc895b7f7d3 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/6fc03c1ffa29406cae3eafc895b7f7d3 2024-11-26T20:42:36,958 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/b344a3c4590241b4a7cc6ca6d7b3d6f8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/C/b344a3c4590241b4a7cc6ca6d7b3d6f8 2024-11-26T20:42:36,960 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/recovered.edits/289.seqid to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332/recovered.edits/289.seqid 2024-11-26T20:42:36,961 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:36,961 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T20:42:36,961 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T20:42:36,962 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-26T20:42:36,963 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260707b1f4a7c644f8afc22a00897f3c52_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260707b1f4a7c644f8afc22a00897f3c52_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:36,964 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126285abf256a0e490aa4b62d4273ec3886_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126285abf256a0e490aa4b62d4273ec3886_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:36,965 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126290c3739a3e74956a062c6f1838bbb36_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126290c3739a3e74956a062c6f1838bbb36_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:36,966 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112654fc9b61af6c46cd8a80c04bfa7e1190_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112654fc9b61af6c46cd8a80c04bfa7e1190_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:36,967 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265a6cb512c3714f9e94cef2f5d6000998_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265a6cb512c3714f9e94cef2f5d6000998_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:36,967 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126603a154d18ab4cc585de4399ab33dd09_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126603a154d18ab4cc585de4399ab33dd09_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:36,968 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112668473e32afac4ebeb93e73b02e499418_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112668473e32afac4ebeb93e73b02e499418_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:36,969 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112673887274496340318ea27fadcd915a6c_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112673887274496340318ea27fadcd915a6c_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:36,970 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126743efd4f6bf9452e9a5147a2e7016e1b_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126743efd4f6bf9452e9a5147a2e7016e1b_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:36,971 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126772e4e9efb8f4ad8a6bd60c930d7a903_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126772e4e9efb8f4ad8a6bd60c930d7a903_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:36,971 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112680dfc5cdf69c46e896ee4cd36cdc0bcd_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112680dfc5cdf69c46e896ee4cd36cdc0bcd_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:36,972 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126819efedfa3754f5c97b1b165197d87ff_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126819efedfa3754f5c97b1b165197d87ff_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:36,973 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126840c380cc00d406ca22bdb2b9fad88e2_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126840c380cc00d406ca22bdb2b9fad88e2_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:36,974 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411268a1fd9d0301c4e879e26f4b15e64acaf_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411268a1fd9d0301c4e879e26f4b15e64acaf_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:36,975 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b0a6ce931d99426ab4bf935428357a0c_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b0a6ce931d99426ab4bf935428357a0c_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:36,975 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126cc14787ce0cf4046be864b6264163701_1591a955d2fc691174b77e9b19a3d332 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126cc14787ce0cf4046be864b6264163701_1591a955d2fc691174b77e9b19a3d332 2024-11-26T20:42:36,976 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T20:42:36,978 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=119, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:42:36,979 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T20:42:36,981 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T20:42:36,982 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=119, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:42:36,982 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T20:42:36,982 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732653756982"}]},"ts":"9223372036854775807"} 2024-11-26T20:42:36,983 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T20:42:36,983 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1591a955d2fc691174b77e9b19a3d332, NAME => 'TestAcidGuarantees,,1732653727381.1591a955d2fc691174b77e9b19a3d332.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T20:42:36,983 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T20:42:36,984 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732653756983"}]},"ts":"9223372036854775807"} 2024-11-26T20:42:36,985 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T20:42:36,985 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=119, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:42:36,986 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 40 msec 2024-11-26T20:42:37,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T20:42:37,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T20:42:37,054 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:42:37,063 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1445524182_22 at /127.0.0.1:55378 [Waiting for operation #1045] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1770967530_22 at /127.0.0.1:55556 [Waiting for operation #837] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/cd6be161543e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1770967530_22 at /127.0.0.1:55548 [Waiting for operation #823] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1445524182_22 at /127.0.0.1:55540 [Waiting for operation #841] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=380 (was 402), ProcessCount=11 (was 11), AvailableMemoryMB=5244 (was 5762) 2024-11-26T20:42:37,071 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=380, ProcessCount=11, AvailableMemoryMB=5243 2024-11-26T20:42:37,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T20:42:37,073 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T20:42:37,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T20:42:37,075 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T20:42:37,075 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:37,075 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 120 2024-11-26T20:42:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-26T20:42:37,075 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T20:42:37,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742254_1430 (size=1061) 2024-11-26T20:42:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-26T20:42:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-26T20:42:37,481 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794 2024-11-26T20:42:37,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742255_1431 (size=53) 2024-11-26T20:42:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-26T20:42:37,886 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:42:37,887 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing eaff56e8e250c06f0a9f8447be8c76bc, disabling compactions & flushes 2024-11-26T20:42:37,887 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:37,887 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:37,887 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. after waiting 0 ms 2024-11-26T20:42:37,887 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:37,887 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:37,887 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for eaff56e8e250c06f0a9f8447be8c76bc: Waiting for close lock at 1732653757886Disabling compacts and flushes for region at 1732653757886Disabling writes for close at 1732653757887 (+1 ms)Writing region close event to WAL at 1732653757887Closed at 1732653757887 2024-11-26T20:42:37,888 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T20:42:37,888 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732653757888"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732653757888"}]},"ts":"1732653757888"} 2024-11-26T20:42:37,889 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T20:42:37,890 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T20:42:37,890 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653757890"}]},"ts":"1732653757890"} 2024-11-26T20:42:37,891 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T20:42:37,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eaff56e8e250c06f0a9f8447be8c76bc, ASSIGN}] 2024-11-26T20:42:37,892 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eaff56e8e250c06f0a9f8447be8c76bc, ASSIGN 2024-11-26T20:42:37,892 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eaff56e8e250c06f0a9f8447be8c76bc, ASSIGN; state=OFFLINE, location=cd6be161543e,40291,1732653631795; forceNewPlan=false, retain=false 2024-11-26T20:42:38,043 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=eaff56e8e250c06f0a9f8447be8c76bc, regionState=OPENING, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:42:38,044 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eaff56e8e250c06f0a9f8447be8c76bc, ASSIGN because future has completed 2024-11-26T20:42:38,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795}] 2024-11-26T20:42:38,198 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:38,198 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7752): Opening region: {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:42:38,199 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:38,199 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:42:38,199 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7794): checking encryption for eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:38,199 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7797): checking classloading for eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:38,200 INFO [StoreOpener-eaff56e8e250c06f0a9f8447be8c76bc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:38,201 INFO [StoreOpener-eaff56e8e250c06f0a9f8447be8c76bc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:42:38,201 INFO [StoreOpener-eaff56e8e250c06f0a9f8447be8c76bc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaff56e8e250c06f0a9f8447be8c76bc columnFamilyName A 2024-11-26T20:42:38,202 DEBUG [StoreOpener-eaff56e8e250c06f0a9f8447be8c76bc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:38,202 INFO [StoreOpener-eaff56e8e250c06f0a9f8447be8c76bc-1 {}] regionserver.HStore(327): Store=eaff56e8e250c06f0a9f8447be8c76bc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:42:38,202 INFO [StoreOpener-eaff56e8e250c06f0a9f8447be8c76bc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:38,203 INFO [StoreOpener-eaff56e8e250c06f0a9f8447be8c76bc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:42:38,203 INFO [StoreOpener-eaff56e8e250c06f0a9f8447be8c76bc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaff56e8e250c06f0a9f8447be8c76bc columnFamilyName B 2024-11-26T20:42:38,203 DEBUG [StoreOpener-eaff56e8e250c06f0a9f8447be8c76bc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:38,203 INFO [StoreOpener-eaff56e8e250c06f0a9f8447be8c76bc-1 {}] regionserver.HStore(327): Store=eaff56e8e250c06f0a9f8447be8c76bc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:42:38,203 INFO [StoreOpener-eaff56e8e250c06f0a9f8447be8c76bc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:38,204 INFO [StoreOpener-eaff56e8e250c06f0a9f8447be8c76bc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:42:38,204 INFO [StoreOpener-eaff56e8e250c06f0a9f8447be8c76bc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaff56e8e250c06f0a9f8447be8c76bc columnFamilyName C 2024-11-26T20:42:38,204 DEBUG [StoreOpener-eaff56e8e250c06f0a9f8447be8c76bc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:42:38,205 INFO [StoreOpener-eaff56e8e250c06f0a9f8447be8c76bc-1 {}] regionserver.HStore(327): Store=eaff56e8e250c06f0a9f8447be8c76bc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:42:38,205 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:38,205 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1038): replaying wal for eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:38,205 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:38,205 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:38,206 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1048): stopping wal replay for eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:38,206 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1060): Cleaning up temporary data for eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:38,206 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T20:42:38,207 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1093): writing seq id for eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:38,209 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:42:38,209 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1114): Opened eaff56e8e250c06f0a9f8447be8c76bc; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66313462, jitterRate=-0.011852413415908813}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T20:42:38,209 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:38,209 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1006): Region open journal for eaff56e8e250c06f0a9f8447be8c76bc: Running coprocessor pre-open hook at 1732653758199Writing region info on filesystem at 1732653758199Initializing all the Stores at 1732653758200 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653758200Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653758200Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653758200Cleaning up temporary data from old regions at 1732653758206 (+6 ms)Running coprocessor post-open hooks at 1732653758209 (+3 ms)Region opened successfully at 1732653758209 2024-11-26T20:42:38,210 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., pid=122, masterSystemTime=1732653758196 2024-11-26T20:42:38,212 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:38,212 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:38,212 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=eaff56e8e250c06f0a9f8447be8c76bc, regionState=OPEN, openSeqNum=2, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:42:38,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 because future has completed 2024-11-26T20:42:38,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-26T20:42:38,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-26T20:42:38,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; OpenRegionProcedure eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 in 170 msec 2024-11-26T20:42:38,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-26T20:42:38,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eaff56e8e250c06f0a9f8447be8c76bc, ASSIGN in 324 msec 2024-11-26T20:42:38,217 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T20:42:38,217 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653758217"}]},"ts":"1732653758217"} 2024-11-26T20:42:38,218 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T20:42:38,219 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T20:42:38,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1460 sec 2024-11-26T20:42:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-26T20:42:39,224 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:42:39,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be1708d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,225 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:42:39,225 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:42:39,226 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:42:39,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:42:39,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:42:39,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f92ccfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:42:39,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:42:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:39,227 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:42:39,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25da33b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,228 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:42:39,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:42:39,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:42:39,230 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:42:39,230 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T20:42:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is cd6be161543e,42359,1732653630959 2024-11-26T20:42:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@44aa0b25 2024-11-26T20:42:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T20:42:39,232 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T20:42:39,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fe0696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,233 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:42:39,233 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:42:39,233 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:42:39,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:42:39,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:42:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c0eecea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:42:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:42:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:39,234 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:42:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59aeb1ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59de117f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,235 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:42:39,235 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:42:39,236 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:42:39,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:42:39,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:42:39,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@458c4bbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:42:39,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:42:39,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:39,237 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:42:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77f8809a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472a8168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,238 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:42:39,238 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:42:39,238 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:42:39,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:42:39,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:42:39,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1be7b68b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:42:39,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:42:39,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:39,239 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:42:39,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7271d93f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f010827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,240 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:42:39,240 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:42:39,240 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:42:39,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:42:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:42:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1322ffff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:42:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:42:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:39,241 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:42:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@147cf077, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2822c33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,242 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:42:39,242 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:42:39,243 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:42:39,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:42:39,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:42:39,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f21f4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:42:39,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:42:39,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:39,244 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:42:39,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78fb0238, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc5e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,245 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:42:39,245 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:42:39,245 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:42:39,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:42:39,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:42:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d79e606, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:42:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:42:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:39,246 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:42:39,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@310ad9df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42daa6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,247 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:42:39,247 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:42:39,248 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:42:39,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:42:39,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:42:39,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f074962, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:42:39,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:42:39,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:39,249 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:42:39,249 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@639b7332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,250 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,250 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:42:39,250 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:42:39,250 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:42:39,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:42:39,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:42:39,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3728be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:42:39,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:42:39,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:39,251 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:42:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@144a1ec0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e8e518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,252 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:42:39,252 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:42:39,253 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:42:39,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:42:39,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:42:39,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6793b816, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:42:39,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:42:39,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:39,254 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:42:39,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68e114f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ff7be5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,255 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:42:39,255 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:42:39,255 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:42:39,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:42:39,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:42:39,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ab9d2c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:42:39,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:42:39,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:39,256 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:42:39,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a032169, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:42:39,257 DEBUG [Thread-1945 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:42:39,257 DEBUG [Thread-1947 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:42:39,257 DEBUG [Thread-1962 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:42:39,257 DEBUG [Thread-1954 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:42:39,257 DEBUG [Thread-1960 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:42:39,258 DEBUG [Thread-1951 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:42:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:42:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:42:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:42:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:42:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:42:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:42:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:42:39,258 DEBUG [Thread-1956 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:42:39,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:42:39,259 DEBUG [Thread-1943 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:42:39,259 DEBUG [Thread-1958 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:42:39,259 DEBUG [Thread-1949 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:42:39,259 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:42:39,259 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:42:39,260 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:42:39,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:42:39,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:42:39,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:42:39,260 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:42:39,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:42:39,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:42:39,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:42:39,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:42:39,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:42:39,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:42:39,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:42:39,261 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:42:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-26T20:42:39,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:42:39,262 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:42:39,262 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:42:39,263 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:42:39,263 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:42:39,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:42:39,263 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:42:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T20:42:39,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:42:39,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:42:39,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:42:39,263 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:42:39,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:42:39,264 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:42:39,264 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:42:39,264 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:42:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:42:39,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:42:39,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:42:39,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:42:39,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:42:39,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2] 2024-11-26T20:42:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:39,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaff56e8e250c06f0a9f8447be8c76bc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T20:42:39,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=A 2024-11-26T20:42:39,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:39,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=B 2024-11-26T20:42:39,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:39,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=C 2024-11-26T20:42:39,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:39,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/d796fd99527d44bbbd2a2df801f74a05 is 50, key is test_row_0/A:col10/1732653759272/Put/seqid=0 2024-11-26T20:42:39,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742256_1432 (size=12001) 2024-11-26T20:42:39,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732653759291, max timestamp=1732653759308Num uniques 859; 2024-11-26T20:42:39,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-26T20:42:39,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732653759291, max timestamp=1732653759308Num uniques 859; 2024-11-26T20:42:39,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-26T20:42:39,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732653759291, max timestamp=1732653759308Num uniques 859; 2024-11-26T20:42:39,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-26T20:42:39,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653819341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:39,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:39,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653819343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:39,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:39,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:39,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653819344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:39,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:39,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653819345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:39,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:39,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653819347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:39,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T20:42:39,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T20:42:39,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:39,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:39,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:39,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:39,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:39,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:39,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653819464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:39,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653819464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:39,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653819464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:39,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653819464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:39,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653819464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:39,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T20:42:39,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:39,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:39,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:39,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:39,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:39,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T20:42:39,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:39,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653819684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:39,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:39,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653819684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:39,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653819684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653819684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:39,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653819684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:39,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:39,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:39,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/d796fd99527d44bbbd2a2df801f74a05 2024-11-26T20:42:39,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T20:42:39,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:39,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:39,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:39,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:39,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:39,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/de848da6cebe456f8d3bd678eba05ca9 is 50, key is test_row_0/B:col10/1732653759272/Put/seqid=0 2024-11-26T20:42:39,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742257_1433 (size=12001) 2024-11-26T20:42:39,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T20:42:39,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:39,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:39,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:39,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:39,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T20:42:40,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653820004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:40,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653820004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:40,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653820004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:40,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:40,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:40,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653820004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:40,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:40,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:40,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653820004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:40,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:40,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T20:42:40,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:40,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:40,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:40,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:40,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:40,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/de848da6cebe456f8d3bd678eba05ca9 2024-11-26T20:42:40,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/8ce1525c74f4430aae3bb20586689c2c is 50, key is test_row_0/C:col10/1732653759272/Put/seqid=0 2024-11-26T20:42:40,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T20:42:40,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:40,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:40,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:40,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:40,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:40,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742258_1434 (size=12001) 2024-11-26T20:42:40,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/8ce1525c74f4430aae3bb20586689c2c 2024-11-26T20:42:40,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/d796fd99527d44bbbd2a2df801f74a05 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/d796fd99527d44bbbd2a2df801f74a05 2024-11-26T20:42:40,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/d796fd99527d44bbbd2a2df801f74a05, entries=150, sequenceid=11, filesize=11.7 K 2024-11-26T20:42:40,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/de848da6cebe456f8d3bd678eba05ca9 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/de848da6cebe456f8d3bd678eba05ca9 2024-11-26T20:42:40,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/de848da6cebe456f8d3bd678eba05ca9, entries=150, sequenceid=11, filesize=11.7 K 2024-11-26T20:42:40,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/8ce1525c74f4430aae3bb20586689c2c as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/8ce1525c74f4430aae3bb20586689c2c 2024-11-26T20:42:40,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/8ce1525c74f4430aae3bb20586689c2c, entries=150, sequenceid=11, filesize=11.7 K 2024-11-26T20:42:40,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for eaff56e8e250c06f0a9f8447be8c76bc in 962ms, sequenceid=11, compaction requested=false 2024-11-26T20:42:40,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:40,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T20:42:40,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:40,341 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing eaff56e8e250c06f0a9f8447be8c76bc 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-26T20:42:40,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=A 2024-11-26T20:42:40,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:42:40,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=B 2024-11-26T20:42:40,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:42:40,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=C 2024-11-26T20:42:40,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:42:40,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/a65d87126fce49d9b94e41c6ef96c8d3 is 50, key is test_row_0/A:col10/1732653759342/Put/seqid=0 2024-11-26T20:42:40,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742259_1435 (size=12001) 2024-11-26T20:42:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T20:42:40,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:40,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:40,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653820532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:40,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653820533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:40,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653820534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:40,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:40,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:40,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653820539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:40,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:40,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:40,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653820545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:40,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:40,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:40,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653820654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:40,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:40,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653820654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:40,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:40,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653820654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:40,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:40,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653820654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:40,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:40,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:40,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653820664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:40,759 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/a65d87126fce49d9b94e41c6ef96c8d3 2024-11-26T20:42:40,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/d0642dde2dcd4bd68c24f251a36f65e8 is 50, key is test_row_0/B:col10/1732653759342/Put/seqid=0 2024-11-26T20:42:40,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742260_1436 (size=12001) 2024-11-26T20:42:40,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653820874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:40,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653820874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653820874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:40,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653820874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:40,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653820883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:41,169 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/d0642dde2dcd4bd68c24f251a36f65e8 2024-11-26T20:42:41,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/c1deaef79e6e42cd81e4e37972646494 is 50, key is test_row_0/C:col10/1732653759342/Put/seqid=0 2024-11-26T20:42:41,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742261_1437 (size=12001) 2024-11-26T20:42:41,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:41,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653821193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:41,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:41,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653821194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:41,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:41,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653821194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:41,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:41,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653821194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:41,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:41,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653821204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:41,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T20:42:41,581 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/c1deaef79e6e42cd81e4e37972646494 2024-11-26T20:42:41,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/a65d87126fce49d9b94e41c6ef96c8d3 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/a65d87126fce49d9b94e41c6ef96c8d3 2024-11-26T20:42:41,587 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/a65d87126fce49d9b94e41c6ef96c8d3, entries=150, sequenceid=45, filesize=11.7 K 2024-11-26T20:42:41,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/d0642dde2dcd4bd68c24f251a36f65e8 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/d0642dde2dcd4bd68c24f251a36f65e8 2024-11-26T20:42:41,591 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/d0642dde2dcd4bd68c24f251a36f65e8, entries=150, sequenceid=45, filesize=11.7 K 2024-11-26T20:42:41,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/c1deaef79e6e42cd81e4e37972646494 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/c1deaef79e6e42cd81e4e37972646494 2024-11-26T20:42:41,594 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/c1deaef79e6e42cd81e4e37972646494, entries=150, sequenceid=45, filesize=11.7 K 2024-11-26T20:42:41,595 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for eaff56e8e250c06f0a9f8447be8c76bc in 1254ms, sequenceid=45, compaction requested=false 2024-11-26T20:42:41,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:41,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:41,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-26T20:42:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-26T20:42:41,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-26T20:42:41,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3330 sec 2024-11-26T20:42:41,601 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.3380 sec 2024-11-26T20:42:41,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:41,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaff56e8e250c06f0a9f8447be8c76bc 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T20:42:41,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=A 2024-11-26T20:42:41,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:41,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=B 2024-11-26T20:42:41,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:41,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=C 2024-11-26T20:42:41,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:41,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/9cf4ad50ad6c43449269b058c8a8908b is 50, key is test_row_0/A:col10/1732653761708/Put/seqid=0 2024-11-26T20:42:41,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742262_1438 (size=14341) 2024-11-26T20:42:41,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653821746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:41,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732653761733, max timestamp=1732653761753Num uniques 859; 2024-11-26T20:42:41,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732653761733, max timestamp=1732653761753Num uniques 859; 2024-11-26T20:42:41,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T20:42:41,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732653761733, max timestamp=1732653761753Num uniques 859; 2024-11-26T20:42:41,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T20:42:41,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T20:42:41,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653821790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:41,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653821790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:41,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653821791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:41,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653821793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:41,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653821863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:41,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653821914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:41,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:41,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653821914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:41,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:41,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653821914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:41,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653821914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:41,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:41,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:41,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:42,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653822084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:42,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/9cf4ad50ad6c43449269b058c8a8908b 2024-11-26T20:42:42,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/82bb843892704aa08e7147627e928551 is 50, key is test_row_0/B:col10/1732653761708/Put/seqid=0 2024-11-26T20:42:42,125 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T20:42:42,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742263_1439 (size=12001) 2024-11-26T20:42:42,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653822134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:42,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:42,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653822134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653822134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:42,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653822135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:42,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:42,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653822404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:42,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:42,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653822454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:42,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:42,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653822454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:42,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:42,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653822454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:42,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653822454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:42,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/82bb843892704aa08e7147627e928551 2024-11-26T20:42:42,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/0b15e94c3c0947b39675ddfa056ffc17 is 50, key is test_row_0/C:col10/1732653761708/Put/seqid=0 2024-11-26T20:42:42,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742264_1440 (size=12001) 2024-11-26T20:42:42,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/0b15e94c3c0947b39675ddfa056ffc17 2024-11-26T20:42:42,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/9cf4ad50ad6c43449269b058c8a8908b as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/9cf4ad50ad6c43449269b058c8a8908b 2024-11-26T20:42:42,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/9cf4ad50ad6c43449269b058c8a8908b, entries=200, sequenceid=56, filesize=14.0 K 2024-11-26T20:42:42,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/82bb843892704aa08e7147627e928551 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/82bb843892704aa08e7147627e928551 2024-11-26T20:42:42,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/82bb843892704aa08e7147627e928551, entries=150, sequenceid=56, filesize=11.7 K 2024-11-26T20:42:42,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/0b15e94c3c0947b39675ddfa056ffc17 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0b15e94c3c0947b39675ddfa056ffc17 2024-11-26T20:42:42,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0b15e94c3c0947b39675ddfa056ffc17, entries=150, sequenceid=56, filesize=11.7 K 2024-11-26T20:42:42,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for eaff56e8e250c06f0a9f8447be8c76bc in 844ms, sequenceid=56, compaction requested=true 2024-11-26T20:42:42,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:42,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:42:42,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:42,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:42:42,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:42,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:42:42,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:42,553 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:42,553 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:42,556 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:42,556 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:42,556 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/A is initiating minor compaction (all files) 2024-11-26T20:42:42,556 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/B is initiating minor compaction (all files) 2024-11-26T20:42:42,556 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/A in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:42,556 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/B in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:42,556 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/d796fd99527d44bbbd2a2df801f74a05, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/a65d87126fce49d9b94e41c6ef96c8d3, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/9cf4ad50ad6c43449269b058c8a8908b] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=37.4 K 2024-11-26T20:42:42,556 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/de848da6cebe456f8d3bd678eba05ca9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/d0642dde2dcd4bd68c24f251a36f65e8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/82bb843892704aa08e7147627e928551] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=35.2 K 2024-11-26T20:42:42,556 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting d796fd99527d44bbbd2a2df801f74a05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732653759269 2024-11-26T20:42:42,556 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting de848da6cebe456f8d3bd678eba05ca9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732653759269 2024-11-26T20:42:42,556 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting a65d87126fce49d9b94e41c6ef96c8d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732653759324 2024-11-26T20:42:42,556 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting d0642dde2dcd4bd68c24f251a36f65e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732653759324 2024-11-26T20:42:42,557 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cf4ad50ad6c43449269b058c8a8908b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732653760533 2024-11-26T20:42:42,557 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 82bb843892704aa08e7147627e928551, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732653760538 2024-11-26T20:42:42,564 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#A#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:42,564 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/9f36f5732d944b2e8cce1df77fa02242 is 50, key is test_row_0/A:col10/1732653761708/Put/seqid=0 2024-11-26T20:42:42,564 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#B#compaction#373 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:42,565 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/89b9738b4211436e95af3b106d9ada18 is 50, key is test_row_0/B:col10/1732653761708/Put/seqid=0 2024-11-26T20:42:42,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742265_1441 (size=12104) 2024-11-26T20:42:42,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742266_1442 (size=12104) 2024-11-26T20:42:42,578 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/89b9738b4211436e95af3b106d9ada18 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/89b9738b4211436e95af3b106d9ada18 2024-11-26T20:42:42,582 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/B of eaff56e8e250c06f0a9f8447be8c76bc into 89b9738b4211436e95af3b106d9ada18(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:42,583 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:42,583 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/B, priority=13, startTime=1732653762553; duration=0sec 2024-11-26T20:42:42,583 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:42,583 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:B 2024-11-26T20:42:42,583 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:42,583 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:42,583 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/C is initiating minor compaction (all files) 2024-11-26T20:42:42,583 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/C in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:42,584 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/8ce1525c74f4430aae3bb20586689c2c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/c1deaef79e6e42cd81e4e37972646494, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0b15e94c3c0947b39675ddfa056ffc17] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=35.2 K 2024-11-26T20:42:42,584 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ce1525c74f4430aae3bb20586689c2c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732653759269 2024-11-26T20:42:42,584 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting c1deaef79e6e42cd81e4e37972646494, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732653759324 2024-11-26T20:42:42,586 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b15e94c3c0947b39675ddfa056ffc17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732653760538 2024-11-26T20:42:42,595 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#C#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:42,596 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/e88e7b73a3cb40aebba784acd71a0976 is 50, key is test_row_0/C:col10/1732653761708/Put/seqid=0 2024-11-26T20:42:42,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742267_1443 (size=12104) 2024-11-26T20:42:42,610 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/e88e7b73a3cb40aebba784acd71a0976 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/e88e7b73a3cb40aebba784acd71a0976 2024-11-26T20:42:42,614 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/C of eaff56e8e250c06f0a9f8447be8c76bc into e88e7b73a3cb40aebba784acd71a0976(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:42,614 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:42,614 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/C, priority=13, startTime=1732653762553; duration=0sec 2024-11-26T20:42:42,614 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:42,614 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:C 2024-11-26T20:42:42,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:42,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaff56e8e250c06f0a9f8447be8c76bc 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-26T20:42:42,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=A 2024-11-26T20:42:42,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:42:42,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=B 2024-11-26T20:42:42,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:42:42,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=C 2024-11-26T20:42:42,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:42:42,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/ad5fbea536ac47fea18400c8ff9c5301 is 50, key is test_row_0/A:col10/1732653762929/Put/seqid=0 2024-11-26T20:42:42,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742268_1444 (size=14341) 2024-11-26T20:42:42,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653822964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:42,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653822967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,974 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/9f36f5732d944b2e8cce1df77fa02242 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/9f36f5732d944b2e8cce1df77fa02242 2024-11-26T20:42:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:42,978 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/A of eaff56e8e250c06f0a9f8447be8c76bc into 9f36f5732d944b2e8cce1df77fa02242(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:42,978 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:42,978 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/A, priority=13, startTime=1732653762553; duration=0sec 2024-11-26T20:42:42,978 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:42,978 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:A 2024-11-26T20:42:42,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653822971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:42,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:42,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653822972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:42,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653822974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:43,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653823084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:43,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653823084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:43,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653823094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653823094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:43,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:43,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653823304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:43,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:43,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653823304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:43,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:43,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:43,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653823304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:43,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:43,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:43,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:43,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653823304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:43,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:43,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/ad5fbea536ac47fea18400c8ff9c5301 2024-11-26T20:42:43,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/ad73584dc5674626988af4bb18e7e005 is 50, key is test_row_0/B:col10/1732653762929/Put/seqid=0 2024-11-26T20:42:43,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742269_1445 (size=12001) 2024-11-26T20:42:43,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/ad73584dc5674626988af4bb18e7e005 2024-11-26T20:42:43,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/0ddcc400f5d8463e886cc642c04494fd is 50, key is test_row_0/C:col10/1732653762929/Put/seqid=0 2024-11-26T20:42:43,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742270_1446 (size=12001) 2024-11-26T20:42:43,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T20:42:43,425 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:42:43,425 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:42:43,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-26T20:42:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T20:42:43,427 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:42:43,428 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:42:43,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:42:43,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T20:42:43,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T20:42:43,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:43,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:43,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:43,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:43,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:43,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:43,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:43,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653823624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:43,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653823624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:43,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653823624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653823624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:43,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T20:42:43,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:43,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:43,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:43,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:43,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T20:42:43,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/0ddcc400f5d8463e886cc642c04494fd 2024-11-26T20:42:43,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/ad5fbea536ac47fea18400c8ff9c5301 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/ad5fbea536ac47fea18400c8ff9c5301 2024-11-26T20:42:43,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/ad5fbea536ac47fea18400c8ff9c5301, entries=200, sequenceid=93, filesize=14.0 K 2024-11-26T20:42:43,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/ad73584dc5674626988af4bb18e7e005 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/ad73584dc5674626988af4bb18e7e005 2024-11-26T20:42:43,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/ad73584dc5674626988af4bb18e7e005, entries=150, sequenceid=93, filesize=11.7 K 2024-11-26T20:42:43,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/0ddcc400f5d8463e886cc642c04494fd as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0ddcc400f5d8463e886cc642c04494fd 2024-11-26T20:42:43,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0ddcc400f5d8463e886cc642c04494fd, entries=150, sequenceid=93, filesize=11.7 K 2024-11-26T20:42:43,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for eaff56e8e250c06f0a9f8447be8c76bc in 854ms, sequenceid=93, compaction requested=false 2024-11-26T20:42:43,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:43,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T20:42:43,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:43,887 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing eaff56e8e250c06f0a9f8447be8c76bc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T20:42:43,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=A 2024-11-26T20:42:43,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:43,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=B 2024-11-26T20:42:43,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:43,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=C 2024-11-26T20:42:43,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:43,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/6539927b3f5b438daeb6e9c143bf1d65 is 50, key is test_row_0/A:col10/1732653762971/Put/seqid=0 2024-11-26T20:42:43,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742271_1447 (size=12001) 2024-11-26T20:42:43,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:43,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T20:42:44,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653764084, max timestamp=1732653764119Num uniques 859; 2024-11-26T20:42:44,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:42:44,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653764084, max timestamp=1732653764119Num uniques 859; 2024-11-26T20:42:44,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653764084, max timestamp=1732653764119Num uniques 859; 2024-11-26T20:42:44,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:42:44,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:42:44,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653824176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653824181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:44,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653824183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:44,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653824191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:44,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653824192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:44,296 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/6539927b3f5b438daeb6e9c143bf1d65 2024-11-26T20:42:44,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/0c37710946e6469a8f5f606f44d494a0 is 50, key is test_row_0/B:col10/1732653762971/Put/seqid=0 2024-11-26T20:42:44,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742272_1448 (size=12001) 2024-11-26T20:42:44,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653824305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:44,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653824306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:44,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653824306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653824306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:44,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653824314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:44,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653824524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:44,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:44,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:44,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653824524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:44,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:44,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:44,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653824524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:44,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:44,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653824524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:44,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:44,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:44,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653824524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:44,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T20:42:44,707 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/0c37710946e6469a8f5f606f44d494a0 2024-11-26T20:42:44,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/c23c51181798404ca0bfeb381f24a576 is 50, key is test_row_0/C:col10/1732653762971/Put/seqid=0 2024-11-26T20:42:44,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742273_1449 (size=12001) 2024-11-26T20:42:44,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:44,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653824844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:44,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:44,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:44,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653824844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:44,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:44,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653824844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:44,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:44,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:44,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653824844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:44,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:44,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653824844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:44,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:44,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:44,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:44,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:45,120 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/c23c51181798404ca0bfeb381f24a576 2024-11-26T20:42:45,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/6539927b3f5b438daeb6e9c143bf1d65 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/6539927b3f5b438daeb6e9c143bf1d65 2024-11-26T20:42:45,127 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/6539927b3f5b438daeb6e9c143bf1d65, entries=150, sequenceid=104, filesize=11.7 K 2024-11-26T20:42:45,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/0c37710946e6469a8f5f606f44d494a0 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/0c37710946e6469a8f5f606f44d494a0 2024-11-26T20:42:45,131 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/0c37710946e6469a8f5f606f44d494a0, entries=150, sequenceid=104, filesize=11.7 K 2024-11-26T20:42:45,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/c23c51181798404ca0bfeb381f24a576 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/c23c51181798404ca0bfeb381f24a576 2024-11-26T20:42:45,135 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/c23c51181798404ca0bfeb381f24a576, entries=150, sequenceid=104, filesize=11.7 K 2024-11-26T20:42:45,136 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for eaff56e8e250c06f0a9f8447be8c76bc in 1249ms, sequenceid=104, compaction requested=true 2024-11-26T20:42:45,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:45,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:45,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-26T20:42:45,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-26T20:42:45,139 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-26T20:42:45,139 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7090 sec 2024-11-26T20:42:45,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.7150 sec 2024-11-26T20:42:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:45,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaff56e8e250c06f0a9f8447be8c76bc 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-26T20:42:45,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=A 2024-11-26T20:42:45,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T20:42:45,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=B 2024-11-26T20:42:45,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T20:42:45,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=C 2024-11-26T20:42:45,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T20:42:45,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/8181856e3a32497bba4a5b31aad781af is 50, key is test_row_0/A:col10/1732653764155/Put/seqid=0 2024-11-26T20:42:45,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742274_1450 (size=14541) 2024-11-26T20:42:45,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653825376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:45,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653825378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:45,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:45,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653825378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:45,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:45,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653825386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:45,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653825387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:45,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:45,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653825494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:45,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653825494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:45,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653825494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:45,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:45,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653825494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:45,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:45,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653825504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:45,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T20:42:45,584 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:42:45,585 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:42:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-26T20:42:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T20:42:45,586 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:42:45,587 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:42:45,587 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:42:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T20:42:45,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653825714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:45,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653825714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:45,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:45,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:45,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653825714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:45,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:45,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:45,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653825714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:45,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:45,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653825724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:45,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-26T20:42:45,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:45,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:45,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:45,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:45,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:45,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/8181856e3a32497bba4a5b31aad781af 2024-11-26T20:42:45,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/11c4fdc390aa463084d94a71b8a95929 is 50, key is test_row_0/B:col10/1732653764155/Put/seqid=0 2024-11-26T20:42:45,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742275_1451 (size=12151) 2024-11-26T20:42:45,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-26T20:42:45,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:45,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:45,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:45,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:45,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T20:42:46,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:46,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653826024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:46,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:46,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653826034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653826034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653826034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:46,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-26T20:42:46,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:46,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:46,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:46,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:46,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:46,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653826043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:46,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/11c4fdc390aa463084d94a71b8a95929 2024-11-26T20:42:46,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-26T20:42:46,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:46,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:46,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:46,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:46,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:46,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/318f8dbc6ab5436eb0f4e901a59612c8 is 50, key is test_row_0/C:col10/1732653764155/Put/seqid=0 2024-11-26T20:42:46,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742276_1452 (size=12151) 2024-11-26T20:42:46,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T20:42:46,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/318f8dbc6ab5436eb0f4e901a59612c8 2024-11-26T20:42:46,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/8181856e3a32497bba4a5b31aad781af as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/8181856e3a32497bba4a5b31aad781af 2024-11-26T20:42:46,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/8181856e3a32497bba4a5b31aad781af, entries=200, sequenceid=142, filesize=14.2 K 2024-11-26T20:42:46,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/11c4fdc390aa463084d94a71b8a95929 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/11c4fdc390aa463084d94a71b8a95929 2024-11-26T20:42:46,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/11c4fdc390aa463084d94a71b8a95929, entries=150, sequenceid=142, filesize=11.9 K 2024-11-26T20:42:46,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/318f8dbc6ab5436eb0f4e901a59612c8 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/318f8dbc6ab5436eb0f4e901a59612c8 2024-11-26T20:42:46,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/318f8dbc6ab5436eb0f4e901a59612c8, entries=150, sequenceid=142, filesize=11.9 K 2024-11-26T20:42:46,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for eaff56e8e250c06f0a9f8447be8c76bc in 864ms, sequenceid=142, compaction requested=true 2024-11-26T20:42:46,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:46,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:42:46,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:46,235 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:42:46,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:42:46,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:46,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:42:46,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:46,235 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:42:46,236 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:42:46,236 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/B is initiating minor compaction (all files) 2024-11-26T20:42:46,236 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/B in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:46,237 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/89b9738b4211436e95af3b106d9ada18, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/ad73584dc5674626988af4bb18e7e005, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/0c37710946e6469a8f5f606f44d494a0, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/11c4fdc390aa463084d94a71b8a95929] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=47.1 K 2024-11-26T20:42:46,237 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52987 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:42:46,237 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/A is initiating minor compaction (all files) 2024-11-26T20:42:46,237 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/A in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:46,237 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/9f36f5732d944b2e8cce1df77fa02242, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/ad5fbea536ac47fea18400c8ff9c5301, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/6539927b3f5b438daeb6e9c143bf1d65, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/8181856e3a32497bba4a5b31aad781af] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=51.7 K 2024-11-26T20:42:46,237 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 89b9738b4211436e95af3b106d9ada18, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732653760538 2024-11-26T20:42:46,237 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f36f5732d944b2e8cce1df77fa02242, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732653760538 2024-11-26T20:42:46,238 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting ad73584dc5674626988af4bb18e7e005, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732653761788 2024-11-26T20:42:46,238 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad5fbea536ac47fea18400c8ff9c5301, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732653761788 2024-11-26T20:42:46,238 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c37710946e6469a8f5f606f44d494a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732653762954 2024-11-26T20:42:46,238 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6539927b3f5b438daeb6e9c143bf1d65, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732653762954 2024-11-26T20:42:46,239 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 11c4fdc390aa463084d94a71b8a95929, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732653764155 2024-11-26T20:42:46,241 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8181856e3a32497bba4a5b31aad781af, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732653764155 2024-11-26T20:42:46,253 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#A#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:46,254 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/fd4b4b47f9ad47249c84b56f5065720b is 50, key is test_row_0/A:col10/1732653764155/Put/seqid=0 2024-11-26T20:42:46,255 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#B#compaction#385 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:46,256 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/1605d4fd906a41049298b3c2a2d94e70 is 50, key is test_row_0/B:col10/1732653764155/Put/seqid=0 2024-11-26T20:42:46,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742278_1454 (size=12391) 2024-11-26T20:42:46,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742277_1453 (size=12391) 2024-11-26T20:42:46,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-26T20:42:46,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:46,351 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing eaff56e8e250c06f0a9f8447be8c76bc 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-26T20:42:46,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=A 2024-11-26T20:42:46,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:46,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=B 2024-11-26T20:42:46,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:46,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=C 2024-11-26T20:42:46,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:46,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/e6e862b6073d4a7b9b80eb7fe7e249da is 50, key is test_row_0/A:col10/1732653765376/Put/seqid=0 2024-11-26T20:42:46,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742279_1455 (size=12151) 2024-11-26T20:42:46,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:46,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:46,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653766581, max timestamp=1732653766598Num uniques 859; 2024-11-26T20:42:46,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653766581, max timestamp=1732653766598Num uniques 859; 2024-11-26T20:42:46,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:42:46,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:42:46,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653766581, max timestamp=1732653766598Num uniques 859; 2024-11-26T20:42:46,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:42:46,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653826646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:46,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653826647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:46,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:46,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:46,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653826648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:46,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:46,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653826648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:46,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:46,675 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/fd4b4b47f9ad47249c84b56f5065720b as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/fd4b4b47f9ad47249c84b56f5065720b 2024-11-26T20:42:46,675 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/1605d4fd906a41049298b3c2a2d94e70 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/1605d4fd906a41049298b3c2a2d94e70 2024-11-26T20:42:46,679 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/A of eaff56e8e250c06f0a9f8447be8c76bc into fd4b4b47f9ad47249c84b56f5065720b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:46,679 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:46,679 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/A, priority=12, startTime=1732653766235; duration=0sec 2024-11-26T20:42:46,679 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:46,679 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:A 2024-11-26T20:42:46,679 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/B of eaff56e8e250c06f0a9f8447be8c76bc into 1605d4fd906a41049298b3c2a2d94e70(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:46,679 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:46,680 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/B, priority=12, startTime=1732653766235; duration=0sec 2024-11-26T20:42:46,680 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:42:46,680 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:46,680 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:B 2024-11-26T20:42:46,680 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:42:46,680 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/C is initiating minor compaction (all files) 2024-11-26T20:42:46,680 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/C in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:46,681 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/e88e7b73a3cb40aebba784acd71a0976, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0ddcc400f5d8463e886cc642c04494fd, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/c23c51181798404ca0bfeb381f24a576, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/318f8dbc6ab5436eb0f4e901a59612c8] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=47.1 K 2024-11-26T20:42:46,681 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting e88e7b73a3cb40aebba784acd71a0976, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732653760538 2024-11-26T20:42:46,681 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ddcc400f5d8463e886cc642c04494fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732653761788 2024-11-26T20:42:46,681 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting c23c51181798404ca0bfeb381f24a576, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732653762954 2024-11-26T20:42:46,681 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 318f8dbc6ab5436eb0f4e901a59612c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732653764155 2024-11-26T20:42:46,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:46,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653826664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:46,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:46,688 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#C#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:46,689 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/bc2be58dd28b42b0b2de8c7a098fe982 is 50, key is test_row_0/C:col10/1732653764155/Put/seqid=0 2024-11-26T20:42:46,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742280_1456 (size=12391) 2024-11-26T20:42:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T20:42:46,759 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/e6e862b6073d4a7b9b80eb7fe7e249da 2024-11-26T20:42:46,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/504f047e05ac46708719136097211c80 is 50, key is test_row_0/B:col10/1732653765376/Put/seqid=0 2024-11-26T20:42:46,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742281_1457 (size=12151) 2024-11-26T20:42:46,769 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/504f047e05ac46708719136097211c80 2024-11-26T20:42:46,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/8399bf6d73454518b0c07dca7854b72a is 50, key is test_row_0/C:col10/1732653765376/Put/seqid=0 2024-11-26T20:42:46,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742282_1458 (size=12151) 2024-11-26T20:42:46,781 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/8399bf6d73454518b0c07dca7854b72a 2024-11-26T20:42:46,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/e6e862b6073d4a7b9b80eb7fe7e249da as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/e6e862b6073d4a7b9b80eb7fe7e249da 2024-11-26T20:42:46,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:46,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653826774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:46,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:46,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653826774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:46,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:46,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653826774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:46,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:46,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653826774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:46,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:46,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:46,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:46,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:46,790 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/e6e862b6073d4a7b9b80eb7fe7e249da, entries=150, sequenceid=151, filesize=11.9 K 2024-11-26T20:42:46,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/504f047e05ac46708719136097211c80 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/504f047e05ac46708719136097211c80 2024-11-26T20:42:46,794 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/504f047e05ac46708719136097211c80, entries=150, sequenceid=151, filesize=11.9 K 2024-11-26T20:42:46,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/8399bf6d73454518b0c07dca7854b72a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/8399bf6d73454518b0c07dca7854b72a 2024-11-26T20:42:46,797 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/8399bf6d73454518b0c07dca7854b72a, entries=150, sequenceid=151, filesize=11.9 K 2024-11-26T20:42:46,798 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for eaff56e8e250c06f0a9f8447be8c76bc in 447ms, sequenceid=151, compaction requested=false 2024-11-26T20:42:46,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:46,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:46,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-26T20:42:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-26T20:42:46,801 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-26T20:42:46,801 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2120 sec 2024-11-26T20:42:46,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.2170 sec 2024-11-26T20:42:46,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:46,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732653766643, max timestamp=1732653766804Num uniques 1068; 2024-11-26T20:42:46,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaff56e8e250c06f0a9f8447be8c76bc 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-26T20:42:46,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-26T20:42:46,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=A 2024-11-26T20:42:46,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:42:46,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=B 2024-11-26T20:42:46,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T20:42:46,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=C 2024-11-26T20:42:46,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T20:42:46,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/f2dc31b78dd54c3c8410f7e4f10cc763 is 50, key is test_row_0/A:col10/1732653766643/Put/seqid=0 2024-11-26T20:42:46,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742283_1459 (size=14541) 2024-11-26T20:42:46,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:46,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653826938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:46,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:46,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:47,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:47,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653826994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:47,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:47,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653826994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:47,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:47,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:47,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653826994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:47,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:47,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:47,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:47,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653826994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:47,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:47,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:47,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653827054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:47,100 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/bc2be58dd28b42b0b2de8c7a098fe982 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/bc2be58dd28b42b0b2de8c7a098fe982 2024-11-26T20:42:47,104 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/C of eaff56e8e250c06f0a9f8447be8c76bc into bc2be58dd28b42b0b2de8c7a098fe982(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:47,104 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:47,104 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/C, priority=12, startTime=1732653766235; duration=0sec 2024-11-26T20:42:47,104 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:47,104 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:C 2024-11-26T20:42:47,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/f2dc31b78dd54c3c8410f7e4f10cc763 2024-11-26T20:42:47,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/8b76bbf61b254f21b21f673b598359a7 is 50, key is test_row_0/B:col10/1732653766643/Put/seqid=0 2024-11-26T20:42:47,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742284_1460 (size=12151) 2024-11-26T20:42:47,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/8b76bbf61b254f21b21f673b598359a7 2024-11-26T20:42:47,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/d6b57f30c1cb40608fee638aeb5ca7f5 is 50, key is test_row_0/C:col10/1732653766643/Put/seqid=0 2024-11-26T20:42:47,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742285_1461 (size=12151) 2024-11-26T20:42:47,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:47,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653827274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:47,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:47,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653827313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:47,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:47,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653827314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:47,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:47,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:47,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653827314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:47,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:47,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:47,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653827314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:47,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:47,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:47,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653827584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:47,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:47,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/d6b57f30c1cb40608fee638aeb5ca7f5 2024-11-26T20:42:47,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/f2dc31b78dd54c3c8410f7e4f10cc763 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/f2dc31b78dd54c3c8410f7e4f10cc763 2024-11-26T20:42:47,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/f2dc31b78dd54c3c8410f7e4f10cc763, entries=200, sequenceid=191, filesize=14.2 K 2024-11-26T20:42:47,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/8b76bbf61b254f21b21f673b598359a7 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/8b76bbf61b254f21b21f673b598359a7 2024-11-26T20:42:47,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/8b76bbf61b254f21b21f673b598359a7, entries=150, sequenceid=191, filesize=11.9 K 2024-11-26T20:42:47,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/d6b57f30c1cb40608fee638aeb5ca7f5 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/d6b57f30c1cb40608fee638aeb5ca7f5 2024-11-26T20:42:47,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/d6b57f30c1cb40608fee638aeb5ca7f5, entries=150, sequenceid=191, filesize=11.9 K 2024-11-26T20:42:47,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~302.31 KB/309568, currentSize=73.80 KB/75570 for eaff56e8e250c06f0a9f8447be8c76bc in 855ms, sequenceid=191, compaction requested=true 2024-11-26T20:42:47,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:47,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:42:47,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:47,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:42:47,664 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:47,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:47,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:42:47,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:47,664 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:47,665 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39083 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:47,665 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:47,666 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/B is initiating minor compaction (all files) 2024-11-26T20:42:47,666 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/A is initiating minor compaction (all files) 2024-11-26T20:42:47,666 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/B in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:47,666 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/A in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:47,666 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/1605d4fd906a41049298b3c2a2d94e70, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/504f047e05ac46708719136097211c80, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/8b76bbf61b254f21b21f673b598359a7] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=35.8 K 2024-11-26T20:42:47,666 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/fd4b4b47f9ad47249c84b56f5065720b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/e6e862b6073d4a7b9b80eb7fe7e249da, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/f2dc31b78dd54c3c8410f7e4f10cc763] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=38.2 K 2024-11-26T20:42:47,666 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd4b4b47f9ad47249c84b56f5065720b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732653764155 2024-11-26T20:42:47,666 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 1605d4fd906a41049298b3c2a2d94e70, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732653764155 2024-11-26T20:42:47,666 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 504f047e05ac46708719136097211c80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732653765375 2024-11-26T20:42:47,666 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6e862b6073d4a7b9b80eb7fe7e249da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732653765375 2024-11-26T20:42:47,667 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2dc31b78dd54c3c8410f7e4f10cc763, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732653766643 2024-11-26T20:42:47,667 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b76bbf61b254f21b21f673b598359a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732653766643 2024-11-26T20:42:47,673 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#B#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:47,673 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#A#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:47,674 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/fb7d754d39fd42079dc0af0bbe553e37 is 50, key is test_row_0/B:col10/1732653766643/Put/seqid=0 2024-11-26T20:42:47,674 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/65b4b179b03445748a41eb86905fc99a is 50, key is test_row_0/A:col10/1732653766643/Put/seqid=0 2024-11-26T20:42:47,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742286_1462 (size=12493) 2024-11-26T20:42:47,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742287_1463 (size=12493) 2024-11-26T20:42:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T20:42:47,734 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:42:47,735 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:42:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-26T20:42:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T20:42:47,737 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:42:47,738 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:42:47,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:42:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:47,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaff56e8e250c06f0a9f8447be8c76bc 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T20:42:47,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=A 2024-11-26T20:42:47,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:47,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=B 2024-11-26T20:42:47,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:47,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=C 2024-11-26T20:42:47,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T20:42:47,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/05f28ed83dd74527aecf15a32e1eb625 is 50, key is test_row_0/A:col10/1732653766876/Put/seqid=0 2024-11-26T20:42:47,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742288_1464 (size=14541) 2024-11-26T20:42:47,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653827865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:47,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:47,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653827866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:47,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:47,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:47,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653827874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:47,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653827875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:47,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:47,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:47,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T20:42:47,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:47,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:47,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:47,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:47,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:47,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653827984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:47,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653827984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:48,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:48,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653827994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:48,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:48,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:48,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653827994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:48,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:48,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:48,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:48,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T20:42:48,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:48,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:48,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:48,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:48,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:48,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T20:42:48,083 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/fb7d754d39fd42079dc0af0bbe553e37 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/fb7d754d39fd42079dc0af0bbe553e37 2024-11-26T20:42:48,084 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/65b4b179b03445748a41eb86905fc99a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/65b4b179b03445748a41eb86905fc99a 2024-11-26T20:42:48,087 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/B of eaff56e8e250c06f0a9f8447be8c76bc into fb7d754d39fd42079dc0af0bbe553e37(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:48,087 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:48,087 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/B, priority=13, startTime=1732653767664; duration=0sec 2024-11-26T20:42:48,087 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:48,087 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:B 2024-11-26T20:42:48,087 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:48,088 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/A of eaff56e8e250c06f0a9f8447be8c76bc into 65b4b179b03445748a41eb86905fc99a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:48,088 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:48,088 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/A, priority=13, startTime=1732653767664; duration=0sec 2024-11-26T20:42:48,088 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:48,088 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:A 2024-11-26T20:42:48,088 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:48,088 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/C is initiating minor compaction (all files) 2024-11-26T20:42:48,088 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/C in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:48,088 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/bc2be58dd28b42b0b2de8c7a098fe982, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/8399bf6d73454518b0c07dca7854b72a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/d6b57f30c1cb40608fee638aeb5ca7f5] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=35.8 K 2024-11-26T20:42:48,089 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting bc2be58dd28b42b0b2de8c7a098fe982, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732653764155 2024-11-26T20:42:48,089 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 8399bf6d73454518b0c07dca7854b72a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732653765375 2024-11-26T20:42:48,089 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting d6b57f30c1cb40608fee638aeb5ca7f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732653766643 2024-11-26T20:42:48,095 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#C#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:48,096 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/effe8c71e5d24629a58052a41d47d018 is 50, key is test_row_0/C:col10/1732653766643/Put/seqid=0 2024-11-26T20:42:48,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742289_1465 (size=12493) 2024-11-26T20:42:48,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653828104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:48,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T20:42:48,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:48,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:48,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:48,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:48,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:48,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653828204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:48,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653828204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:48,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653828213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:48,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653828213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:48,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/05f28ed83dd74527aecf15a32e1eb625 2024-11-26T20:42:48,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/fd832a83f49449a89fcf42c347e1d686 is 50, key is test_row_0/B:col10/1732653766876/Put/seqid=0 2024-11-26T20:42:48,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742290_1466 (size=12151) 2024-11-26T20:42:48,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T20:42:48,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:48,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:48,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:48,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:48,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T20:42:48,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T20:42:48,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:48,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:48,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:48,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:48,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:48,505 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/effe8c71e5d24629a58052a41d47d018 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/effe8c71e5d24629a58052a41d47d018 2024-11-26T20:42:48,509 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/C of eaff56e8e250c06f0a9f8447be8c76bc into effe8c71e5d24629a58052a41d47d018(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:48,509 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:48,509 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/C, priority=13, startTime=1732653767664; duration=0sec 2024-11-26T20:42:48,509 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:48,509 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:C 2024-11-26T20:42:48,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653828524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:48,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653828524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:48,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:48,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:48,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:48,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:48,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:48,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:48,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653828533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:48,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:48,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:48,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653828533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:48,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:48,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:48,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:48,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:48,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T20:42:48,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:48,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:48,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:48,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:48,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:48,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/fd832a83f49449a89fcf42c347e1d686 2024-11-26T20:42:48,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/60240ab7c3a846429f8fd1a42ac29bbf is 50, key is test_row_0/C:col10/1732653766876/Put/seqid=0 2024-11-26T20:42:48,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742291_1467 (size=12151) 2024-11-26T20:42:48,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T20:42:48,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:48,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:48,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:48,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:48,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:48,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:48,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T20:42:48,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T20:42:48,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:48,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:48,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:48,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:48,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:49,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653829034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:49,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653829043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:49,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:49,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653829053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:49,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:49,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653829054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:49,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:49,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:49,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:49,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:49,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:49,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:49,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/60240ab7c3a846429f8fd1a42ac29bbf 2024-11-26T20:42:49,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/05f28ed83dd74527aecf15a32e1eb625 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/05f28ed83dd74527aecf15a32e1eb625 2024-11-26T20:42:49,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/05f28ed83dd74527aecf15a32e1eb625, entries=200, sequenceid=207, filesize=14.2 K 2024-11-26T20:42:49,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/fd832a83f49449a89fcf42c347e1d686 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/fd832a83f49449a89fcf42c347e1d686 2024-11-26T20:42:49,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/fd832a83f49449a89fcf42c347e1d686, entries=150, sequenceid=207, filesize=11.9 K 2024-11-26T20:42:49,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/60240ab7c3a846429f8fd1a42ac29bbf as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/60240ab7c3a846429f8fd1a42ac29bbf 2024-11-26T20:42:49,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/60240ab7c3a846429f8fd1a42ac29bbf, entries=150, sequenceid=207, filesize=11.9 K 2024-11-26T20:42:49,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for eaff56e8e250c06f0a9f8447be8c76bc in 1251ms, sequenceid=207, compaction requested=false 2024-11-26T20:42:49,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:49,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T20:42:49,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:49,115 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing eaff56e8e250c06f0a9f8447be8c76bc 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T20:42:49,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=A 2024-11-26T20:42:49,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:49,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=B 2024-11-26T20:42:49,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:49,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=C 2024-11-26T20:42:49,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:49,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/3e4308d7a91f4104b0ec8657eb51d146 is 50, key is test_row_0/A:col10/1732653767874/Put/seqid=0 2024-11-26T20:42:49,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742292_1468 (size=12151) 2024-11-26T20:42:49,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:49,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:49,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:49,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653829207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:49,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:49,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653829324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:49,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:49,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:49,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:49,524 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/3e4308d7a91f4104b0ec8657eb51d146 2024-11-26T20:42:49,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/83f70ac2bbeb44e28787907b2ee8dd27 is 50, key is test_row_0/B:col10/1732653767874/Put/seqid=0 2024-11-26T20:42:49,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742293_1469 (size=12151) 2024-11-26T20:42:49,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653829543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:49,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:49,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:49,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:49,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653829864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:49,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T20:42:49,934 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/83f70ac2bbeb44e28787907b2ee8dd27 2024-11-26T20:42:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/6f23d9ec05824faba6af3dddf8c1fe3a is 50, key is test_row_0/C:col10/1732653767874/Put/seqid=0 2024-11-26T20:42:49,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742294_1470 (size=12151) 2024-11-26T20:42:50,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653830053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:50,063 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in eaff56e8e250c06f0a9f8447be8c76bc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:42:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:50,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653830064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:50,073 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in eaff56e8e250c06f0a9f8447be8c76bc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:42:50,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:50,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:50,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:50,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653830074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:50,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653830074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:50,083 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in eaff56e8e250c06f0a9f8447be8c76bc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:42:50,083 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in eaff56e8e250c06f0a9f8447be8c76bc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:42:50,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:50,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:50,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:50,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:50,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:50,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:50,344 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/6f23d9ec05824faba6af3dddf8c1fe3a 2024-11-26T20:42:50,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/3e4308d7a91f4104b0ec8657eb51d146 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/3e4308d7a91f4104b0ec8657eb51d146 2024-11-26T20:42:50,356 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/3e4308d7a91f4104b0ec8657eb51d146, entries=150, sequenceid=227, filesize=11.9 K 2024-11-26T20:42:50,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/83f70ac2bbeb44e28787907b2ee8dd27 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/83f70ac2bbeb44e28787907b2ee8dd27 2024-11-26T20:42:50,363 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/83f70ac2bbeb44e28787907b2ee8dd27, entries=150, sequenceid=227, filesize=11.9 K 2024-11-26T20:42:50,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/6f23d9ec05824faba6af3dddf8c1fe3a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/6f23d9ec05824faba6af3dddf8c1fe3a 2024-11-26T20:42:50,368 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/6f23d9ec05824faba6af3dddf8c1fe3a, entries=150, sequenceid=227, filesize=11.9 K 2024-11-26T20:42:50,368 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for eaff56e8e250c06f0a9f8447be8c76bc in 1253ms, sequenceid=227, compaction requested=true 2024-11-26T20:42:50,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:50,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:50,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-26T20:42:50,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-26T20:42:50,371 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-26T20:42:50,371 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6320 sec 2024-11-26T20:42:50,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.6370 sec 2024-11-26T20:42:50,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:50,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaff56e8e250c06f0a9f8447be8c76bc 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T20:42:50,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=A 2024-11-26T20:42:50,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:50,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=B 2024-11-26T20:42:50,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:50,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=C 2024-11-26T20:42:50,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:50,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/879e3bff75a84d6d9aa571f65e7bd325 is 50, key is test_row_0/A:col10/1732653769205/Put/seqid=0 2024-11-26T20:42:50,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742295_1471 (size=14541) 2024-11-26T20:42:50,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:50,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653830512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:50,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:50,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653830633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:50,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:50,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:50,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:50,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/879e3bff75a84d6d9aa571f65e7bd325 2024-11-26T20:42:50,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/0645a12e57c14015ab356c0383f3141a is 50, key is test_row_0/B:col10/1732653769205/Put/seqid=0 2024-11-26T20:42:50,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742296_1472 (size=12151) 2024-11-26T20:42:50,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653830854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:50,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:50,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:50,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:51,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653831174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:51,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:51,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:51,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:51,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/0645a12e57c14015ab356c0383f3141a 2024-11-26T20:42:51,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/e8bc74178e404fc3838ae7d718b2c6e0 is 50, key is test_row_0/C:col10/1732653769205/Put/seqid=0 2024-11-26T20:42:51,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742297_1473 (size=12151) 2024-11-26T20:42:51,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/e8bc74178e404fc3838ae7d718b2c6e0 2024-11-26T20:42:51,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/879e3bff75a84d6d9aa571f65e7bd325 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/879e3bff75a84d6d9aa571f65e7bd325 2024-11-26T20:42:51,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/879e3bff75a84d6d9aa571f65e7bd325, entries=200, sequenceid=243, filesize=14.2 K 2024-11-26T20:42:51,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/0645a12e57c14015ab356c0383f3141a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/0645a12e57c14015ab356c0383f3141a 2024-11-26T20:42:51,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/0645a12e57c14015ab356c0383f3141a, entries=150, sequenceid=243, filesize=11.9 K 2024-11-26T20:42:51,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/e8bc74178e404fc3838ae7d718b2c6e0 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/e8bc74178e404fc3838ae7d718b2c6e0 2024-11-26T20:42:51,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/e8bc74178e404fc3838ae7d718b2c6e0, entries=150, sequenceid=243, filesize=11.9 K 2024-11-26T20:42:51,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for eaff56e8e250c06f0a9f8447be8c76bc in 1244ms, sequenceid=243, compaction requested=true 2024-11-26T20:42:51,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:51,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:42:51,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:51,642 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:42:51,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:42:51,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:51,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:42:51,642 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:42:51,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:51,643 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:42:51,643 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/A is initiating minor compaction (all files) 2024-11-26T20:42:51,643 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:42:51,643 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/A in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:51,643 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/B is initiating minor compaction (all files) 2024-11-26T20:42:51,643 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/65b4b179b03445748a41eb86905fc99a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/05f28ed83dd74527aecf15a32e1eb625, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/3e4308d7a91f4104b0ec8657eb51d146, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/879e3bff75a84d6d9aa571f65e7bd325] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=52.5 K 2024-11-26T20:42:51,643 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/B in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:51,643 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/fb7d754d39fd42079dc0af0bbe553e37, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/fd832a83f49449a89fcf42c347e1d686, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/83f70ac2bbeb44e28787907b2ee8dd27, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/0645a12e57c14015ab356c0383f3141a] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=47.8 K 2024-11-26T20:42:51,643 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65b4b179b03445748a41eb86905fc99a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732653766643 2024-11-26T20:42:51,643 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting fb7d754d39fd42079dc0af0bbe553e37, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732653766643 2024-11-26T20:42:51,644 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05f28ed83dd74527aecf15a32e1eb625, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732653766876 2024-11-26T20:42:51,644 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting fd832a83f49449a89fcf42c347e1d686, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732653766876 2024-11-26T20:42:51,644 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e4308d7a91f4104b0ec8657eb51d146, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732653767853 2024-11-26T20:42:51,644 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 83f70ac2bbeb44e28787907b2ee8dd27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732653767853 2024-11-26T20:42:51,644 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 879e3bff75a84d6d9aa571f65e7bd325, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732653769192 2024-11-26T20:42:51,644 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 0645a12e57c14015ab356c0383f3141a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732653769203 2024-11-26T20:42:51,653 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#A#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:51,653 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#B#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:51,653 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/6cb87efb25bb44828c419636f289e76b is 50, key is test_row_0/A:col10/1732653769205/Put/seqid=0 2024-11-26T20:42:51,653 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/fcdf6b263a8b49d8abd33431efeb1554 is 50, key is test_row_0/B:col10/1732653769205/Put/seqid=0 2024-11-26T20:42:51,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742299_1475 (size=12629) 2024-11-26T20:42:51,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742298_1474 (size=12629) 2024-11-26T20:42:51,661 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/fcdf6b263a8b49d8abd33431efeb1554 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/fcdf6b263a8b49d8abd33431efeb1554 2024-11-26T20:42:51,664 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/B of eaff56e8e250c06f0a9f8447be8c76bc into fcdf6b263a8b49d8abd33431efeb1554(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:51,665 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:51,665 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/B, priority=12, startTime=1732653771642; duration=0sec 2024-11-26T20:42:51,665 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:51,665 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:B 2024-11-26T20:42:51,665 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:42:51,666 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:42:51,666 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/C is initiating minor compaction (all files) 2024-11-26T20:42:51,666 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/C in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:51,666 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/effe8c71e5d24629a58052a41d47d018, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/60240ab7c3a846429f8fd1a42ac29bbf, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/6f23d9ec05824faba6af3dddf8c1fe3a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/e8bc74178e404fc3838ae7d718b2c6e0] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=47.8 K 2024-11-26T20:42:51,666 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting effe8c71e5d24629a58052a41d47d018, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732653766643 2024-11-26T20:42:51,666 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 60240ab7c3a846429f8fd1a42ac29bbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732653766876 2024-11-26T20:42:51,666 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f23d9ec05824faba6af3dddf8c1fe3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732653767853 2024-11-26T20:42:51,667 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting e8bc74178e404fc3838ae7d718b2c6e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732653769203 2024-11-26T20:42:51,672 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#C#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:51,672 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/e2e26842bb4c44d0baffc1294638bcc8 is 50, key is test_row_0/C:col10/1732653769205/Put/seqid=0 2024-11-26T20:42:51,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742300_1476 (size=12629) 2024-11-26T20:42:51,687 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/e2e26842bb4c44d0baffc1294638bcc8 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/e2e26842bb4c44d0baffc1294638bcc8 2024-11-26T20:42:51,691 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/C of eaff56e8e250c06f0a9f8447be8c76bc into e2e26842bb4c44d0baffc1294638bcc8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:51,691 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:51,691 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/C, priority=12, startTime=1732653771642; duration=0sec 2024-11-26T20:42:51,691 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:51,691 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:C 2024-11-26T20:42:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:51,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaff56e8e250c06f0a9f8447be8c76bc 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T20:42:51,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=A 2024-11-26T20:42:51,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:51,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=B 2024-11-26T20:42:51,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:51,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=C 2024-11-26T20:42:51,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:51,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/e9a005c5007942d391d468370ca9f325 is 50, key is test_row_0/A:col10/1732653770510/Put/seqid=0 2024-11-26T20:42:51,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742301_1477 (size=14691) 2024-11-26T20:42:51,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653831779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T20:42:51,894 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:42:51,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:42:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-26T20:42:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T20:42:51,896 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:42:51,897 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:42:51,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:42:51,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653831894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T20:42:52,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T20:42:52,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:52,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:52,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:52,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:52,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:52,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:52,062 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/6cb87efb25bb44828c419636f289e76b as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/6cb87efb25bb44828c419636f289e76b 2024-11-26T20:42:52,066 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/A of eaff56e8e250c06f0a9f8447be8c76bc into 6cb87efb25bb44828c419636f289e76b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:52,066 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:52,066 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/A, priority=12, startTime=1732653771642; duration=0sec 2024-11-26T20:42:52,066 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:52,066 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:A 2024-11-26T20:42:52,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653832073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:52,083 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in eaff56e8e250c06f0a9f8447be8c76bc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:42:52,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:52,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:52,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:52,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:52,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653832094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:52,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:52,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653832094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:52,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:52,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653832094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:52,102 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in eaff56e8e250c06f0a9f8447be8c76bc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:42:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:52,103 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in eaff56e8e250c06f0a9f8447be8c76bc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:42:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:52,103 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in eaff56e8e250c06f0a9f8447be8c76bc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:42:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:52,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/e9a005c5007942d391d468370ca9f325 2024-11-26T20:42:52,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/4dd6ddd6a023414993055d8883adb9b5 is 50, key is test_row_0/B:col10/1732653770510/Put/seqid=0 2024-11-26T20:42:52,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742302_1478 (size=12251) 2024-11-26T20:42:52,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:52,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653832114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:52,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T20:42:52,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:52,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:52,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:52,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:52,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T20:42:52,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T20:42:52,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:52,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:52,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:52,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:52,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:52,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653832434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:52,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:52,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:52,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:52,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T20:42:52,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:52,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:52,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:52,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:52,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:52,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/4dd6ddd6a023414993055d8883adb9b5 2024-11-26T20:42:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T20:42:52,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/6cc5f55302d04de98c5aafdf65748b38 is 50, key is test_row_0/C:col10/1732653770510/Put/seqid=0 2024-11-26T20:42:52,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742303_1479 (size=12251) 2024-11-26T20:42:52,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T20:42:52,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:52,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:52,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:52,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:52,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:52,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T20:42:52,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:52,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:52,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:52,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:52,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:52,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/6cc5f55302d04de98c5aafdf65748b38 2024-11-26T20:42:52,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/e9a005c5007942d391d468370ca9f325 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/e9a005c5007942d391d468370ca9f325 2024-11-26T20:42:52,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/e9a005c5007942d391d468370ca9f325, entries=200, sequenceid=262, filesize=14.3 K 2024-11-26T20:42:52,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/4dd6ddd6a023414993055d8883adb9b5 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/4dd6ddd6a023414993055d8883adb9b5 2024-11-26T20:42:52,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/4dd6ddd6a023414993055d8883adb9b5, entries=150, sequenceid=262, filesize=12.0 K 2024-11-26T20:42:52,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/6cc5f55302d04de98c5aafdf65748b38 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/6cc5f55302d04de98c5aafdf65748b38 2024-11-26T20:42:52,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/6cc5f55302d04de98c5aafdf65748b38, entries=150, sequenceid=262, filesize=12.0 K 2024-11-26T20:42:52,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for eaff56e8e250c06f0a9f8447be8c76bc in 1245ms, sequenceid=262, compaction requested=false 2024-11-26T20:42:52,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:52,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaff56e8e250c06f0a9f8447be8c76bc 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T20:42:52,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=A 2024-11-26T20:42:52,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:52,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=B 2024-11-26T20:42:52,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:52,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=C 2024-11-26T20:42:52,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:52,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/3d2f4e0d875e497f949870bfc204fbd5 is 50, key is test_row_0/A:col10/1732653771769/Put/seqid=0 2024-11-26T20:42:52,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742304_1480 (size=14741) 2024-11-26T20:42:52,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T20:42:52,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:52,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:52,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:52,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/3d2f4e0d875e497f949870bfc204fbd5 2024-11-26T20:42:52,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:52,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:52,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/d7d2b31340204e049f6e4fa8541b979b is 50, key is test_row_0/B:col10/1732653771769/Put/seqid=0 2024-11-26T20:42:52,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742305_1481 (size=12301) 2024-11-26T20:42:52,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/d7d2b31340204e049f6e4fa8541b979b 2024-11-26T20:42:52,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/2a5c65d43f824f8fb154b04ad2f3c6c2 is 50, key is test_row_0/C:col10/1732653771769/Put/seqid=0 2024-11-26T20:42:52,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742306_1482 (size=12301) 2024-11-26T20:42:52,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/2a5c65d43f824f8fb154b04ad2f3c6c2 2024-11-26T20:42:52,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/3d2f4e0d875e497f949870bfc204fbd5 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/3d2f4e0d875e497f949870bfc204fbd5 2024-11-26T20:42:53,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/3d2f4e0d875e497f949870bfc204fbd5, entries=200, sequenceid=279, filesize=14.4 K 2024-11-26T20:42:53,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/d7d2b31340204e049f6e4fa8541b979b as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/d7d2b31340204e049f6e4fa8541b979b 2024-11-26T20:42:53,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/d7d2b31340204e049f6e4fa8541b979b, entries=150, sequenceid=279, filesize=12.0 K 2024-11-26T20:42:53,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/2a5c65d43f824f8fb154b04ad2f3c6c2 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/2a5c65d43f824f8fb154b04ad2f3c6c2 2024-11-26T20:42:53,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/2a5c65d43f824f8fb154b04ad2f3c6c2, entries=150, sequenceid=279, filesize=12.0 K 2024-11-26T20:42:53,019 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=46.96 KB/48090 for eaff56e8e250c06f0a9f8447be8c76bc in 60ms, sequenceid=279, compaction requested=true 2024-11-26T20:42:53,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:53,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:42:53,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:53,019 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:53,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:42:53,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:53,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:42:53,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T20:42:53,019 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:53,020 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:53,020 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/A is initiating minor compaction (all files) 2024-11-26T20:42:53,020 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/A in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:53,020 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/6cb87efb25bb44828c419636f289e76b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/e9a005c5007942d391d468370ca9f325, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/3d2f4e0d875e497f949870bfc204fbd5] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=41.1 K 2024-11-26T20:42:53,020 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37181 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:53,020 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/B is initiating minor compaction (all files) 2024-11-26T20:42:53,021 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/B in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:53,021 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/fcdf6b263a8b49d8abd33431efeb1554, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/4dd6ddd6a023414993055d8883adb9b5, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/d7d2b31340204e049f6e4fa8541b979b] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=36.3 K 2024-11-26T20:42:53,022 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting fcdf6b263a8b49d8abd33431efeb1554, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732653769203 2024-11-26T20:42:53,022 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cb87efb25bb44828c419636f289e76b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732653769203 2024-11-26T20:42:53,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:53,022 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dd6ddd6a023414993055d8883adb9b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732653770450 2024-11-26T20:42:53,023 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting d7d2b31340204e049f6e4fa8541b979b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732653771769 2024-11-26T20:42:53,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaff56e8e250c06f0a9f8447be8c76bc 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T20:42:53,023 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9a005c5007942d391d468370ca9f325, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732653770450 2024-11-26T20:42:53,023 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d2f4e0d875e497f949870bfc204fbd5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732653771748 2024-11-26T20:42:53,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=A 2024-11-26T20:42:53,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:53,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=B 2024-11-26T20:42:53,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:53,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=C 2024-11-26T20:42:53,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:53,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/6e1cfee08ec74e14b0cd0476486e508a is 50, key is test_row_0/A:col10/1732653772995/Put/seqid=0 2024-11-26T20:42:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T20:42:53,046 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#B#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:53,046 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/06a80908b46e4d57a20ccc441b0ced01 is 50, key is test_row_0/B:col10/1732653771769/Put/seqid=0 2024-11-26T20:42:53,046 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#A#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:53,047 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/826a57773d92480f9ab85537780593f4 is 50, key is test_row_0/A:col10/1732653771769/Put/seqid=0 2024-11-26T20:42:53,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742307_1483 (size=14741) 2024-11-26T20:42:53,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742308_1484 (size=12881) 2024-11-26T20:42:53,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742309_1485 (size=12881) 2024-11-26T20:42:53,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T20:42:53,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:53,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:53,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:53,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:53,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:53,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:53,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653773134, max timestamp=1732653773189Num uniques 859; 2024-11-26T20:42:53,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653773134, max timestamp=1732653773189Num uniques 859; 2024-11-26T20:42:53,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653773134, max timestamp=1732653773189Num uniques 859; 2024-11-26T20:42:53,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:42:53,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:42:53,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:42:53,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T20:42:53,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:53,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:53,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:53,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:53,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:53,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:53,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:53,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653833372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:53,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T20:42:53,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:53,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:53,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:53,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:53,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:53,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:53,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/6e1cfee08ec74e14b0cd0476486e508a 2024-11-26T20:42:53,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/60effdacd5724fcab1035a3d3b2b3442 is 50, key is test_row_0/B:col10/1732653772995/Put/seqid=0 2024-11-26T20:42:53,457 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/06a80908b46e4d57a20ccc441b0ced01 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/06a80908b46e4d57a20ccc441b0ced01 2024-11-26T20:42:53,462 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/B of eaff56e8e250c06f0a9f8447be8c76bc into 06a80908b46e4d57a20ccc441b0ced01(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:53,462 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:53,462 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/B, priority=13, startTime=1732653773019; duration=0sec 2024-11-26T20:42:53,462 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:53,462 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:B 2024-11-26T20:42:53,462 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:53,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742310_1486 (size=12301) 2024-11-26T20:42:53,463 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/826a57773d92480f9ab85537780593f4 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/826a57773d92480f9ab85537780593f4 2024-11-26T20:42:53,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/60effdacd5724fcab1035a3d3b2b3442 2024-11-26T20:42:53,463 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37181 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:53,463 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/C is initiating minor compaction (all files) 2024-11-26T20:42:53,463 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/C in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:53,463 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/e2e26842bb4c44d0baffc1294638bcc8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/6cc5f55302d04de98c5aafdf65748b38, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/2a5c65d43f824f8fb154b04ad2f3c6c2] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=36.3 K 2024-11-26T20:42:53,464 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting e2e26842bb4c44d0baffc1294638bcc8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732653769203 2024-11-26T20:42:53,464 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cc5f55302d04de98c5aafdf65748b38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732653770450 2024-11-26T20:42:53,464 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a5c65d43f824f8fb154b04ad2f3c6c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732653771769 2024-11-26T20:42:53,467 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/A of eaff56e8e250c06f0a9f8447be8c76bc into 826a57773d92480f9ab85537780593f4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:53,467 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:53,467 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/A, priority=13, startTime=1732653773019; duration=0sec 2024-11-26T20:42:53,467 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:53,467 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:A 2024-11-26T20:42:53,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/710381c9b00e4ebe8762274c46eb5116 is 50, key is test_row_0/C:col10/1732653772995/Put/seqid=0 2024-11-26T20:42:53,472 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#C#compaction#419 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:53,472 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/353cd00243cb4a7ba28d9c14e36a406e is 50, key is test_row_0/C:col10/1732653771769/Put/seqid=0 2024-11-26T20:42:53,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742311_1487 (size=12301) 2024-11-26T20:42:53,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742312_1488 (size=12881) 2024-11-26T20:42:53,483 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/353cd00243cb4a7ba28d9c14e36a406e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/353cd00243cb4a7ba28d9c14e36a406e 2024-11-26T20:42:53,486 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/C of eaff56e8e250c06f0a9f8447be8c76bc into 353cd00243cb4a7ba28d9c14e36a406e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:53,486 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:53,486 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/C, priority=13, startTime=1732653773019; duration=0sec 2024-11-26T20:42:53,486 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:53,486 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:C 2024-11-26T20:42:53,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:53,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653833494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:53,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T20:42:53,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:53,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:53,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:53,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:53,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:53,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653833714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:53,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:53,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:53,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:53,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T20:42:53,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:53,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:53,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:53,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:53,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:53,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/710381c9b00e4ebe8762274c46eb5116 2024-11-26T20:42:53,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/6e1cfee08ec74e14b0cd0476486e508a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/6e1cfee08ec74e14b0cd0476486e508a 2024-11-26T20:42:53,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/6e1cfee08ec74e14b0cd0476486e508a, entries=200, sequenceid=290, filesize=14.4 K 2024-11-26T20:42:53,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/60effdacd5724fcab1035a3d3b2b3442 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/60effdacd5724fcab1035a3d3b2b3442 2024-11-26T20:42:53,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/60effdacd5724fcab1035a3d3b2b3442, entries=150, sequenceid=290, filesize=12.0 K 2024-11-26T20:42:53,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/710381c9b00e4ebe8762274c46eb5116 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/710381c9b00e4ebe8762274c46eb5116 2024-11-26T20:42:53,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T20:42:53,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:53,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:53,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:53,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/710381c9b00e4ebe8762274c46eb5116, entries=150, sequenceid=290, filesize=12.0 K 2024-11-26T20:42:53,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:53,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:53,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for eaff56e8e250c06f0a9f8447be8c76bc in 865ms, sequenceid=290, compaction requested=false 2024-11-26T20:42:53,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:54,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaff56e8e250c06f0a9f8447be8c76bc 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-26T20:42:54,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=A 2024-11-26T20:42:54,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:42:54,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=B 2024-11-26T20:42:54,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:42:54,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=C 2024-11-26T20:42:54,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:42:54,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/4023b977194f4316a0cc1e3806a5a1bf is 50, key is test_row_0/A:col10/1732653773215/Put/seqid=0 2024-11-26T20:42:54,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T20:42:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:54,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:54,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T20:42:54,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742313_1489 (size=14741) 2024-11-26T20:42:54,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653834091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:54,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:54,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:54,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:54,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T20:42:54,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:54,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:54,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:54,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:54,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:54,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653834204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:54,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:54,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:54,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:54,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T20:42:54,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:54,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:54,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:54,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:54,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:54,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653834414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:54,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:54,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:54,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:54,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/4023b977194f4316a0cc1e3806a5a1bf 2024-11-26T20:42:54,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/d5aaa96b2cda4680809ca0be7ac9f267 is 50, key is test_row_0/B:col10/1732653773215/Put/seqid=0 2024-11-26T20:42:54,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742314_1490 (size=12301) 2024-11-26T20:42:54,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T20:42:54,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:54,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:54,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:54,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:54,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:54,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T20:42:54,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:54,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:54,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:54,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:54,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:54,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653834734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:54,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:54,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:54,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:54,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T20:42:54,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:54,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:54,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:54,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:54,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:42:54,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/d5aaa96b2cda4680809ca0be7ac9f267 2024-11-26T20:42:54,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/0a02eecded57405cbb4cd56c2f5f986e is 50, key is test_row_0/C:col10/1732653773215/Put/seqid=0 2024-11-26T20:42:54,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742315_1491 (size=12301) 2024-11-26T20:42:54,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/0a02eecded57405cbb4cd56c2f5f986e 2024-11-26T20:42:54,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/4023b977194f4316a0cc1e3806a5a1bf as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/4023b977194f4316a0cc1e3806a5a1bf 2024-11-26T20:42:54,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/4023b977194f4316a0cc1e3806a5a1bf, entries=200, sequenceid=330, filesize=14.4 K 2024-11-26T20:42:54,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/d5aaa96b2cda4680809ca0be7ac9f267 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/d5aaa96b2cda4680809ca0be7ac9f267 2024-11-26T20:42:54,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/d5aaa96b2cda4680809ca0be7ac9f267, entries=150, sequenceid=330, filesize=12.0 K 2024-11-26T20:42:54,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/0a02eecded57405cbb4cd56c2f5f986e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0a02eecded57405cbb4cd56c2f5f986e 2024-11-26T20:42:54,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0a02eecded57405cbb4cd56c2f5f986e, entries=150, sequenceid=330, filesize=12.0 K 2024-11-26T20:42:54,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for eaff56e8e250c06f0a9f8447be8c76bc in 857ms, sequenceid=330, compaction requested=true 2024-11-26T20:42:54,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:54,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:42:54,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:54,890 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:54,890 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:54,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:42:54,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:54,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:42:54,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:54,892 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:54,892 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:54,892 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/B is initiating minor compaction (all files) 2024-11-26T20:42:54,892 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/A is initiating minor compaction (all files) 2024-11-26T20:42:54,892 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/B in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:54,892 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/A in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:54,892 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/06a80908b46e4d57a20ccc441b0ced01, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/60effdacd5724fcab1035a3d3b2b3442, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/d5aaa96b2cda4680809ca0be7ac9f267] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=36.6 K 2024-11-26T20:42:54,892 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/826a57773d92480f9ab85537780593f4, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/6e1cfee08ec74e14b0cd0476486e508a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/4023b977194f4316a0cc1e3806a5a1bf] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=41.4 K 2024-11-26T20:42:54,892 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 06a80908b46e4d57a20ccc441b0ced01, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732653771769 2024-11-26T20:42:54,892 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 826a57773d92480f9ab85537780593f4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732653771769 2024-11-26T20:42:54,893 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e1cfee08ec74e14b0cd0476486e508a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732653772982 2024-11-26T20:42:54,893 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 60effdacd5724fcab1035a3d3b2b3442, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732653772982 2024-11-26T20:42:54,893 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4023b977194f4316a0cc1e3806a5a1bf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732653773215 2024-11-26T20:42:54,893 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting d5aaa96b2cda4680809ca0be7ac9f267, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732653773215 2024-11-26T20:42:54,899 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#B#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:54,900 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/3eb88d44c9d945238ddf45393652de11 is 50, key is test_row_0/B:col10/1732653773215/Put/seqid=0 2024-11-26T20:42:54,902 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#A#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:54,903 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/3c6316b9a783472e861820de2c95b1e3 is 50, key is test_row_0/A:col10/1732653773215/Put/seqid=0 2024-11-26T20:42:54,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742316_1492 (size=12983) 2024-11-26T20:42:54,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742317_1493 (size=12983) 2024-11-26T20:42:54,918 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/3c6316b9a783472e861820de2c95b1e3 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/3c6316b9a783472e861820de2c95b1e3 2024-11-26T20:42:54,922 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/A of eaff56e8e250c06f0a9f8447be8c76bc into 3c6316b9a783472e861820de2c95b1e3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:54,922 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:54,922 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/A, priority=13, startTime=1732653774890; duration=0sec 2024-11-26T20:42:54,922 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:54,922 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:A 2024-11-26T20:42:54,922 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:54,923 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:54,923 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/C is initiating minor compaction (all files) 2024-11-26T20:42:54,924 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/C in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:54,924 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/353cd00243cb4a7ba28d9c14e36a406e, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/710381c9b00e4ebe8762274c46eb5116, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0a02eecded57405cbb4cd56c2f5f986e] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=36.6 K 2024-11-26T20:42:54,924 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 353cd00243cb4a7ba28d9c14e36a406e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732653771769 2024-11-26T20:42:54,924 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 710381c9b00e4ebe8762274c46eb5116, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732653772982 2024-11-26T20:42:54,924 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a02eecded57405cbb4cd56c2f5f986e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732653773215 2024-11-26T20:42:54,932 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#C#compaction#425 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:54,932 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/93914cd79fc74bb882fa0bf715f48016 is 50, key is test_row_0/C:col10/1732653773215/Put/seqid=0 2024-11-26T20:42:54,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742318_1494 (size=12983) 2024-11-26T20:42:54,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T20:42:54,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:54,958 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing eaff56e8e250c06f0a9f8447be8c76bc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T20:42:54,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=A 2024-11-26T20:42:54,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:54,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=B 2024-11-26T20:42:54,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:54,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=C 2024-11-26T20:42:54,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:42:54,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/e7531f3ab9034907aee4e8b9f92a1306 is 50, key is test_row_0/A:col10/1732653774076/Put/seqid=0 2024-11-26T20:42:54,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742319_1495 (size=12301) 2024-11-26T20:42:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:55,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. as already flushing 2024-11-26T20:42:55,313 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/3eb88d44c9d945238ddf45393652de11 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/3eb88d44c9d945238ddf45393652de11 2024-11-26T20:42:55,317 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/B of eaff56e8e250c06f0a9f8447be8c76bc into 3eb88d44c9d945238ddf45393652de11(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:55,317 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:55,317 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/B, priority=13, startTime=1732653774890; duration=0sec 2024-11-26T20:42:55,317 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:55,318 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:B 2024-11-26T20:42:55,353 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/93914cd79fc74bb882fa0bf715f48016 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/93914cd79fc74bb882fa0bf715f48016 2024-11-26T20:42:55,358 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/C of eaff56e8e250c06f0a9f8447be8c76bc into 93914cd79fc74bb882fa0bf715f48016(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:55,358 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:55,358 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/C, priority=13, startTime=1732653774890; duration=0sec 2024-11-26T20:42:55,359 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:55,359 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:C 2024-11-26T20:42:55,369 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/e7531f3ab9034907aee4e8b9f92a1306 2024-11-26T20:42:55,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/79d79d800e444d5a9d286304ebc73a70 is 50, key is test_row_0/B:col10/1732653774076/Put/seqid=0 2024-11-26T20:42:55,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742320_1496 (size=12301) 2024-11-26T20:42:55,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653775368, max timestamp=1732653775399Num uniques 859; 2024-11-26T20:42:55,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:42:55,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653775368, max timestamp=1732653775399Num uniques 859; 2024-11-26T20:42:55,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653775368, max timestamp=1732653775399Num uniques 859; 2024-11-26T20:42:55,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:42:55,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:42:55,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653835542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:55,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:55,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653835664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:55,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:55,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:55,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:55,779 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/79d79d800e444d5a9d286304ebc73a70 2024-11-26T20:42:55,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/0df1c81d93c84b05899470d62e2d584c is 50, key is test_row_0/C:col10/1732653774076/Put/seqid=0 2024-11-26T20:42:55,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742321_1497 (size=12301) 2024-11-26T20:42:55,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653835884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:55,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:55,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:55,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:56,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T20:42:56,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50198 deadline: 1732653836104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:56,114 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in eaff56e8e250c06f0a9f8447be8c76bc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:42:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:56,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50240 deadline: 1732653836114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:56,125 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in eaff56e8e250c06f0a9f8447be8c76bc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:42:56,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:56,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:56,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:56,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732653836124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:56,137 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in eaff56e8e250c06f0a9f8447be8c76bc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:42:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:56,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50186 deadline: 1732653836143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:56,156 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in eaff56e8e250c06f0a9f8447be8c76bc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8280 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:42:56,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:56,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:56,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:56,191 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/0df1c81d93c84b05899470d62e2d584c 2024-11-26T20:42:56,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/e7531f3ab9034907aee4e8b9f92a1306 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/e7531f3ab9034907aee4e8b9f92a1306 2024-11-26T20:42:56,198 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/e7531f3ab9034907aee4e8b9f92a1306, entries=150, sequenceid=341, filesize=12.0 K 2024-11-26T20:42:56,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/79d79d800e444d5a9d286304ebc73a70 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/79d79d800e444d5a9d286304ebc73a70 2024-11-26T20:42:56,202 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/79d79d800e444d5a9d286304ebc73a70, entries=150, sequenceid=341, filesize=12.0 K 2024-11-26T20:42:56,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/0df1c81d93c84b05899470d62e2d584c as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0df1c81d93c84b05899470d62e2d584c 2024-11-26T20:42:56,205 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0df1c81d93c84b05899470d62e2d584c, entries=150, sequenceid=341, filesize=12.0 K 2024-11-26T20:42:56,206 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for eaff56e8e250c06f0a9f8447be8c76bc in 1248ms, sequenceid=341, compaction requested=false 2024-11-26T20:42:56,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:56,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:56,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-26T20:42:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-26T20:42:56,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-26T20:42:56,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.3110 sec 2024-11-26T20:42:56,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 4.3160 sec 2024-11-26T20:42:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:56,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732653775491, max timestamp=1732653776211Num uniques 1018; 2024-11-26T20:42:56,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-26T20:42:56,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaff56e8e250c06f0a9f8447be8c76bc 3/3 column families, dataSize=102.87 KB heapSize=292.20 KB 2024-11-26T20:42:56,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732653775491, max timestamp=1732653776211Num uniques 1018; 2024-11-26T20:42:56,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-26T20:42:56,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=A 2024-11-26T20:42:56,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732653775491, max timestamp=1732653776211Num uniques 1018; 2024-11-26T20:42:56,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:42:56,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=B 2024-11-26T20:42:56,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-26T20:42:56,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:42:56,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=C 2024-11-26T20:42:56,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:42:56,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/fb21f78f593c40b9b43c074076c37cf1 is 50, key is test_row_0/A:col10/1732653775527/Put/seqid=0 2024-11-26T20:42:56,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742322_1498 (size=14741) 2024-11-26T20:42:56,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653776271, max timestamp=1732653776356Num uniques 859; 2024-11-26T20:42:56,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:42:56,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653776271, max timestamp=1732653776356Num uniques 859; 2024-11-26T20:42:56,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:42:56,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653776271, max timestamp=1732653776356Num uniques 859; 2024-11-26T20:42:56,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:42:56,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732653776469, max timestamp=1732653776496Num uniques 1018; 2024-11-26T20:42:56,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-26T20:42:56,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732653776469, max timestamp=1732653776496Num uniques 1018; 2024-11-26T20:42:56,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732653776469, max timestamp=1732653776496Num uniques 1018; 2024-11-26T20:42:56,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-26T20:42:56,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-26T20:42:56,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/fb21f78f593c40b9b43c074076c37cf1 2024-11-26T20:42:56,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/34953f64a33543d5bcee919ae2baaff5 is 50, key is test_row_0/B:col10/1732653775527/Put/seqid=0 2024-11-26T20:42:56,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742323_1499 (size=12301) 2024-11-26T20:42:56,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732653776581, max timestamp=1732653776651Num uniques 1027; 2024-11-26T20:42:56,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-26T20:42:56,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732653776581, max timestamp=1732653776651Num uniques 1027; 2024-11-26T20:42:56,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-26T20:42:56,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732653776581, max timestamp=1732653776651Num uniques 1027; 2024-11-26T20:42:56,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-26T20:42:56,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732653776735, max timestamp=1732653776813Num uniques 1036; 2024-11-26T20:42:56,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-26T20:42:56,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732653776735, max timestamp=1732653776813Num uniques 1036; 2024-11-26T20:42:56,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732653776735, max timestamp=1732653776813Num uniques 1036; 2024-11-26T20:42:56,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-26T20:42:56,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-26T20:42:56,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732653776926, max timestamp=1732653776952Num uniques 1045; 2024-11-26T20:42:56,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732653776926, max timestamp=1732653776952Num uniques 1045; 2024-11-26T20:42:56,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-26T20:42:56,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-26T20:42:56,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732653776926, max timestamp=1732653776952Num uniques 1045; 2024-11-26T20:42:56,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-26T20:42:57,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/34953f64a33543d5bcee919ae2baaff5 2024-11-26T20:42:57,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/4eeac7bfaf6146a2be172bde3ef65a24 is 50, key is test_row_0/C:col10/1732653775527/Put/seqid=0 2024-11-26T20:42:57,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742324_1500 (size=12301) 2024-11-26T20:42:57,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732653777064, max timestamp=1732653777115Num uniques 1054; 2024-11-26T20:42:57,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732653777064, max timestamp=1732653777115Num uniques 1054; 2024-11-26T20:42:57,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732653777064, max timestamp=1732653777115Num uniques 1054; 2024-11-26T20:42:57,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-26T20:42:57,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-26T20:42:57,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-26T20:42:57,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732653777238, max timestamp=1732653777292Num uniques 1013; 2024-11-26T20:42:57,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732653777238, max timestamp=1732653777292Num uniques 1013; 2024-11-26T20:42:57,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-26T20:42:57,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-26T20:42:57,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732653777238, max timestamp=1732653777292Num uniques 1013; 2024-11-26T20:42:57,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-26T20:42:57,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732653777347, max timestamp=1732653777436Num uniques 1022; 2024-11-26T20:42:57,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-26T20:42:57,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732653777347, max timestamp=1732653777436Num uniques 1022; 2024-11-26T20:42:57,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-26T20:42:57,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732653777347, max timestamp=1732653777436Num uniques 1022; 2024-11-26T20:42:57,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-26T20:42:57,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/4eeac7bfaf6146a2be172bde3ef65a24 2024-11-26T20:42:57,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/fb21f78f593c40b9b43c074076c37cf1 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/fb21f78f593c40b9b43c074076c37cf1 2024-11-26T20:42:57,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/fb21f78f593c40b9b43c074076c37cf1, entries=200, sequenceid=381, filesize=14.4 K 2024-11-26T20:42:57,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/34953f64a33543d5bcee919ae2baaff5 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/34953f64a33543d5bcee919ae2baaff5 2024-11-26T20:42:57,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/34953f64a33543d5bcee919ae2baaff5, entries=150, sequenceid=381, filesize=12.0 K 2024-11-26T20:42:57,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/4eeac7bfaf6146a2be172bde3ef65a24 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/4eeac7bfaf6146a2be172bde3ef65a24 2024-11-26T20:42:57,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/4eeac7bfaf6146a2be172bde3ef65a24, entries=150, sequenceid=381, filesize=12.0 K 2024-11-26T20:42:57,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~65.34 KB/66912, currentSize=53.67 KB/54960 for eaff56e8e250c06f0a9f8447be8c76bc in 1255ms, sequenceid=381, compaction requested=true 2024-11-26T20:42:57,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:57,472 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:57,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:42:57,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:57,472 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:57,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:42:57,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:57,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:42:57,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:57,473 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:57,473 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/A is initiating minor compaction (all files) 2024-11-26T20:42:57,473 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/A in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:57,473 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/3c6316b9a783472e861820de2c95b1e3, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/e7531f3ab9034907aee4e8b9f92a1306, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/fb21f78f593c40b9b43c074076c37cf1] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=39.1 K 2024-11-26T20:42:57,474 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:57,474 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/B is initiating minor compaction (all files) 2024-11-26T20:42:57,474 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/B in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:57,474 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/3eb88d44c9d945238ddf45393652de11, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/79d79d800e444d5a9d286304ebc73a70, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/34953f64a33543d5bcee919ae2baaff5] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=36.7 K 2024-11-26T20:42:57,474 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c6316b9a783472e861820de2c95b1e3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732653773215 2024-11-26T20:42:57,474 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eb88d44c9d945238ddf45393652de11, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732653773215 2024-11-26T20:42:57,475 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 79d79d800e444d5a9d286304ebc73a70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732653774076 2024-11-26T20:42:57,475 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7531f3ab9034907aee4e8b9f92a1306, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732653774076 2024-11-26T20:42:57,475 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 34953f64a33543d5bcee919ae2baaff5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732653775527 2024-11-26T20:42:57,475 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb21f78f593c40b9b43c074076c37cf1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732653775491 2024-11-26T20:42:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:57,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaff56e8e250c06f0a9f8447be8c76bc 3/3 column families, dataSize=60.38 KB heapSize=164.63 KB 2024-11-26T20:42:57,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=A 2024-11-26T20:42:57,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:42:57,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=B 2024-11-26T20:42:57,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:42:57,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=C 2024-11-26T20:42:57,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:42:57,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/d219d7e1eaa041dcaadc03da57804788 is 50, key is test_row_0/A:col10/1732653777465/Put/seqid=0 2024-11-26T20:42:57,502 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#A#compaction#433 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:57,504 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/28523c20979845bfa53de81dfbdd034b is 50, key is test_row_0/A:col10/1732653775527/Put/seqid=0 2024-11-26T20:42:57,507 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#B#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:57,507 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/fa5385f70797449899fa67ea0353023b is 50, key is test_row_0/B:col10/1732653775527/Put/seqid=0 2024-11-26T20:42:57,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742325_1501 (size=14741) 2024-11-26T20:42:57,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742326_1502 (size=13085) 2024-11-26T20:42:57,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742327_1503 (size=13085) 2024-11-26T20:42:57,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653777638, max timestamp=1732653777688Num uniques 859; 2024-11-26T20:42:57,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:42:57,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653777638, max timestamp=1732653777688Num uniques 859; 2024-11-26T20:42:57,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:42:57,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653777638, max timestamp=1732653777688Num uniques 859; 2024-11-26T20:42:57,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:42:57,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:57,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 396 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653837884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:57,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:57,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:57,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:57,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/d219d7e1eaa041dcaadc03da57804788 2024-11-26T20:42:57,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/909487a9d8d846bfa3b195c3fa38778a is 50, key is test_row_0/B:col10/1732653777465/Put/seqid=0 2024-11-26T20:42:57,924 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/fa5385f70797449899fa67ea0353023b as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/fa5385f70797449899fa67ea0353023b 2024-11-26T20:42:57,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742328_1504 (size=12301) 2024-11-26T20:42:57,934 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/28523c20979845bfa53de81dfbdd034b as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/28523c20979845bfa53de81dfbdd034b 2024-11-26T20:42:57,934 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/B of eaff56e8e250c06f0a9f8447be8c76bc into fa5385f70797449899fa67ea0353023b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:57,934 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:57,934 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/B, priority=13, startTime=1732653777472; duration=0sec 2024-11-26T20:42:57,934 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:57,934 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:B 2024-11-26T20:42:57,934 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:57,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/909487a9d8d846bfa3b195c3fa38778a 2024-11-26T20:42:57,935 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:57,935 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/C is initiating minor compaction (all files) 2024-11-26T20:42:57,935 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/C in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:57,935 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/93914cd79fc74bb882fa0bf715f48016, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0df1c81d93c84b05899470d62e2d584c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/4eeac7bfaf6146a2be172bde3ef65a24] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=36.7 K 2024-11-26T20:42:57,936 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 93914cd79fc74bb882fa0bf715f48016, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732653773215 2024-11-26T20:42:57,938 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 0df1c81d93c84b05899470d62e2d584c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732653774076 2024-11-26T20:42:57,938 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 4eeac7bfaf6146a2be172bde3ef65a24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732653775527 2024-11-26T20:42:57,939 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/A of eaff56e8e250c06f0a9f8447be8c76bc into 28523c20979845bfa53de81dfbdd034b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:57,939 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:57,939 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/A, priority=13, startTime=1732653777471; duration=0sec 2024-11-26T20:42:57,940 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:57,940 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:A 2024-11-26T20:42:57,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/57abf73d0b9e42e18719d8ee1b5b8afe is 50, key is test_row_0/C:col10/1732653777465/Put/seqid=0 2024-11-26T20:42:57,945 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#C#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:57,946 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/7a8784eb382b492ca7ecca0727c6512a is 50, key is test_row_0/C:col10/1732653775527/Put/seqid=0 2024-11-26T20:42:57,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742329_1505 (size=12301) 2024-11-26T20:42:57,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742330_1506 (size=13085) 2024-11-26T20:42:58,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:58,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 397 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653838003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:58,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:58,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 398 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653838224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:58,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:58,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:58,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:58,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/57abf73d0b9e42e18719d8ee1b5b8afe 2024-11-26T20:42:58,354 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/7a8784eb382b492ca7ecca0727c6512a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/7a8784eb382b492ca7ecca0727c6512a 2024-11-26T20:42:58,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/d219d7e1eaa041dcaadc03da57804788 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/d219d7e1eaa041dcaadc03da57804788 2024-11-26T20:42:58,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/d219d7e1eaa041dcaadc03da57804788, entries=200, sequenceid=527, filesize=14.4 K 2024-11-26T20:42:58,358 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/C of eaff56e8e250c06f0a9f8447be8c76bc into 7a8784eb382b492ca7ecca0727c6512a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:42:58,358 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:58,358 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/C, priority=13, startTime=1732653777472; duration=0sec 2024-11-26T20:42:58,358 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:58,358 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:C 2024-11-26T20:42:58,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/909487a9d8d846bfa3b195c3fa38778a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/909487a9d8d846bfa3b195c3fa38778a 2024-11-26T20:42:58,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/909487a9d8d846bfa3b195c3fa38778a, entries=150, sequenceid=527, filesize=12.0 K 2024-11-26T20:42:58,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/57abf73d0b9e42e18719d8ee1b5b8afe as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/57abf73d0b9e42e18719d8ee1b5b8afe 2024-11-26T20:42:58,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/57abf73d0b9e42e18719d8ee1b5b8afe, entries=150, sequenceid=527, filesize=12.0 K 2024-11-26T20:42:58,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~164.58 KB/168528, currentSize=127.47 KB/130530 for eaff56e8e250c06f0a9f8447be8c76bc in 889ms, sequenceid=527, compaction requested=false 2024-11-26T20:42:58,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:58,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:42:58,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaff56e8e250c06f0a9f8447be8c76bc 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-26T20:42:58,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=A 2024-11-26T20:42:58,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:42:58,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=B 2024-11-26T20:42:58,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:42:58,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=C 2024-11-26T20:42:58,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:42:58,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/f72cd2a185554081b32ee669e42ee22c is 50, key is test_row_0/A:col10/1732653777838/Put/seqid=0 2024-11-26T20:42:58,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742331_1507 (size=14741) 2024-11-26T20:42:58,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:58,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 407 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653838608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:58,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:58,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:58,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:58,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 408 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653838734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:58,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:58,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:58,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:58,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:58,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 409 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653838954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:58,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:58,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:58,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:58,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=567 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/f72cd2a185554081b32ee669e42ee22c 2024-11-26T20:42:58,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/c738ffc196e94767a80629309ca7bc6b is 50, key is test_row_0/B:col10/1732653777838/Put/seqid=0 2024-11-26T20:42:58,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742332_1508 (size=12301) 2024-11-26T20:42:59,260 INFO [Thread-1960 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1960. 2024-11-26T20:42:59,260 DEBUG [Thread-1960 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:42:59,260 DEBUG [Thread-1960 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:59,260 DEBUG [Thread-1960 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:59,260 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:42:59,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 410 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653839263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:59,273 INFO [Thread-1956 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1956. 2024-11-26T20:42:59,274 DEBUG [Thread-1956 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:42:59,274 DEBUG [Thread-1956 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:59,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:42:59,274 DEBUG [Thread-1956 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:59,274 INFO [Thread-1954 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1954. 2024-11-26T20:42:59,274 DEBUG [Thread-1954 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:42:59,274 DEBUG [Thread-1954 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:59,274 DEBUG [Thread-1954 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:59,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:42:59,277 INFO [Thread-1958 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1958. 2024-11-26T20:42:59,277 DEBUG [Thread-1958 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:42:59,278 DEBUG [Thread-1958 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:59,278 DEBUG [Thread-1958 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:59,278 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:42:59,283 INFO [Thread-1962 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1962. 2024-11-26T20:42:59,283 DEBUG [Thread-1962 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:42:59,283 DEBUG [Thread-1962 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:59,283 DEBUG [Thread-1962 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:42:59,283 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:42:59,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=567 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/c738ffc196e94767a80629309ca7bc6b 2024-11-26T20:42:59,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/22ff9951166d4d37b2dd53737aff2c71 is 50, key is test_row_0/C:col10/1732653777838/Put/seqid=0 2024-11-26T20:42:59,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742333_1509 (size=12301) 2024-11-26T20:42:59,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=567 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/22ff9951166d4d37b2dd53737aff2c71 2024-11-26T20:42:59,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/f72cd2a185554081b32ee669e42ee22c as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/f72cd2a185554081b32ee669e42ee22c 2024-11-26T20:42:59,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/f72cd2a185554081b32ee669e42ee22c, entries=200, sequenceid=567, filesize=14.4 K 2024-11-26T20:42:59,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/c738ffc196e94767a80629309ca7bc6b as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/c738ffc196e94767a80629309ca7bc6b 2024-11-26T20:42:59,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:42:59,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 411 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50168 deadline: 1732653839784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 2024-11-26T20:42:59,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 , the old value is region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:59,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:42:59,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., hostname=cd6be161543e,40291,1732653631795, seqNum=2 because the exception is null or not the one we care about 2024-11-26T20:42:59,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/c738ffc196e94767a80629309ca7bc6b, entries=150, sequenceid=567, filesize=12.0 K 2024-11-26T20:42:59,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/22ff9951166d4d37b2dd53737aff2c71 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/22ff9951166d4d37b2dd53737aff2c71 2024-11-26T20:42:59,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/22ff9951166d4d37b2dd53737aff2c71, entries=150, sequenceid=567, filesize=12.0 K 2024-11-26T20:42:59,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for eaff56e8e250c06f0a9f8447be8c76bc in 1241ms, sequenceid=567, compaction requested=true 2024-11-26T20:42:59,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:42:59,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:42:59,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:59,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:42:59,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:42:59,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaff56e8e250c06f0a9f8447be8c76bc:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:42:59,790 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:59,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:42:59,790 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:42:59,790 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42567 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:59,790 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:42:59,790 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/A is initiating minor compaction (all files) 2024-11-26T20:42:59,790 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/B is initiating minor compaction (all files) 2024-11-26T20:42:59,790 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/A in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:59,790 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/B in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:42:59,790 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/28523c20979845bfa53de81dfbdd034b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/d219d7e1eaa041dcaadc03da57804788, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/f72cd2a185554081b32ee669e42ee22c] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=41.6 K 2024-11-26T20:42:59,790 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/fa5385f70797449899fa67ea0353023b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/909487a9d8d846bfa3b195c3fa38778a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/c738ffc196e94767a80629309ca7bc6b] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=36.8 K 2024-11-26T20:42:59,791 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28523c20979845bfa53de81dfbdd034b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732653775527 2024-11-26T20:42:59,791 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting fa5385f70797449899fa67ea0353023b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732653775527 2024-11-26T20:42:59,791 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting d219d7e1eaa041dcaadc03da57804788, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1732653777448 2024-11-26T20:42:59,791 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 909487a9d8d846bfa3b195c3fa38778a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1732653777460 2024-11-26T20:42:59,791 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting f72cd2a185554081b32ee669e42ee22c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=567, earliestPutTs=1732653777838 2024-11-26T20:42:59,791 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting c738ffc196e94767a80629309ca7bc6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=567, earliestPutTs=1732653777838 2024-11-26T20:42:59,796 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#B#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:59,796 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/c17033d408074c938934c47bdfaf4731 is 50, key is test_row_0/B:col10/1732653777838/Put/seqid=0 2024-11-26T20:42:59,797 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#A#compaction#442 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:42:59,798 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/50eee35ee8a8455b94c85b0fa1b1774b is 50, key is test_row_0/A:col10/1732653777838/Put/seqid=0 2024-11-26T20:42:59,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742334_1510 (size=13187) 2024-11-26T20:42:59,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742335_1511 (size=13187) 2024-11-26T20:42:59,967 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T20:43:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T20:43:00,084 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:43:00,203 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/c17033d408074c938934c47bdfaf4731 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/c17033d408074c938934c47bdfaf4731 2024-11-26T20:43:00,207 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/B of eaff56e8e250c06f0a9f8447be8c76bc into c17033d408074c938934c47bdfaf4731(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:43:00,207 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:43:00,207 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/B, priority=13, startTime=1732653779790; duration=0sec 2024-11-26T20:43:00,207 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:43:00,207 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:B 2024-11-26T20:43:00,207 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:43:00,208 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:43:00,208 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): eaff56e8e250c06f0a9f8447be8c76bc/C is initiating minor compaction (all files) 2024-11-26T20:43:00,208 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaff56e8e250c06f0a9f8447be8c76bc/C in TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:43:00,208 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/7a8784eb382b492ca7ecca0727c6512a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/57abf73d0b9e42e18719d8ee1b5b8afe, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/22ff9951166d4d37b2dd53737aff2c71] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp, totalSize=36.8 K 2024-11-26T20:43:00,208 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a8784eb382b492ca7ecca0727c6512a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732653775527 2024-11-26T20:43:00,209 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 57abf73d0b9e42e18719d8ee1b5b8afe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1732653777460 2024-11-26T20:43:00,209 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/50eee35ee8a8455b94c85b0fa1b1774b as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/50eee35ee8a8455b94c85b0fa1b1774b 2024-11-26T20:43:00,209 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 22ff9951166d4d37b2dd53737aff2c71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=567, earliestPutTs=1732653777838 2024-11-26T20:43:00,212 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/A of eaff56e8e250c06f0a9f8447be8c76bc into 50eee35ee8a8455b94c85b0fa1b1774b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:43:00,212 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:43:00,212 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/A, priority=13, startTime=1732653779789; duration=0sec 2024-11-26T20:43:00,212 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:00,212 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:A 2024-11-26T20:43:00,214 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaff56e8e250c06f0a9f8447be8c76bc#C#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:43:00,214 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/c3dd4d153414495cbd09a114aa74d37d is 50, key is test_row_0/C:col10/1732653777838/Put/seqid=0 2024-11-26T20:43:00,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742336_1512 (size=13187) 2024-11-26T20:43:00,623 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/c3dd4d153414495cbd09a114aa74d37d as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/c3dd4d153414495cbd09a114aa74d37d 2024-11-26T20:43:00,627 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaff56e8e250c06f0a9f8447be8c76bc/C of eaff56e8e250c06f0a9f8447be8c76bc into c3dd4d153414495cbd09a114aa74d37d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:43:00,627 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:43:00,627 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., storeName=eaff56e8e250c06f0a9f8447be8c76bc/C, priority=13, startTime=1732653779790; duration=0sec 2024-11-26T20:43:00,627 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:00,627 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaff56e8e250c06f0a9f8447be8c76bc:C 2024-11-26T20:43:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:43:00,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaff56e8e250c06f0a9f8447be8c76bc 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T20:43:00,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=A 2024-11-26T20:43:00,797 INFO [Thread-1947 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1947. 2024-11-26T20:43:00,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:00,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=B 2024-11-26T20:43:00,797 DEBUG [Thread-1947 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:43:00,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:00,797 DEBUG [Thread-1947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:00,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=C 2024-11-26T20:43:00,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:00,797 DEBUG [Thread-1947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:00,797 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:43:00,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/28469b7f5e824249ae0849d7d6e40f64 is 50, key is test_row_0/A:col10/1732653778590/Put/seqid=0 2024-11-26T20:43:00,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742337_1513 (size=12301) 2024-11-26T20:43:01,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=581 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/28469b7f5e824249ae0849d7d6e40f64 2024-11-26T20:43:01,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/b2c8b609411f4947bf4e8371898fc907 is 50, key is test_row_0/B:col10/1732653778590/Put/seqid=0 2024-11-26T20:43:01,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742338_1514 (size=12301) 2024-11-26T20:43:01,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=581 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/b2c8b609411f4947bf4e8371898fc907 2024-11-26T20:43:01,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/bcaf7e9422ad4be699bd9617b4277aef is 50, key is test_row_0/C:col10/1732653778590/Put/seqid=0 2024-11-26T20:43:01,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742339_1515 (size=12301) 2024-11-26T20:43:01,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=581 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/bcaf7e9422ad4be699bd9617b4277aef 2024-11-26T20:43:01,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/28469b7f5e824249ae0849d7d6e40f64 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/28469b7f5e824249ae0849d7d6e40f64 2024-11-26T20:43:01,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/28469b7f5e824249ae0849d7d6e40f64, entries=150, sequenceid=581, filesize=12.0 K 2024-11-26T20:43:01,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/b2c8b609411f4947bf4e8371898fc907 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/b2c8b609411f4947bf4e8371898fc907 2024-11-26T20:43:01,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/b2c8b609411f4947bf4e8371898fc907, entries=150, sequenceid=581, filesize=12.0 K 2024-11-26T20:43:01,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/bcaf7e9422ad4be699bd9617b4277aef as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/bcaf7e9422ad4be699bd9617b4277aef 2024-11-26T20:43:01,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/bcaf7e9422ad4be699bd9617b4277aef, entries=150, sequenceid=581, filesize=12.0 K 2024-11-26T20:43:01,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for eaff56e8e250c06f0a9f8447be8c76bc in 859ms, sequenceid=581, compaction requested=false 2024-11-26T20:43:01,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaff56e8e250c06f0a9f8447be8c76bc: 2024-11-26T20:43:06,135 INFO [Thread-1949 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1949. 2024-11-26T20:43:06,136 DEBUG [Thread-1949 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:43:06,136 DEBUG [Thread-1949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:06,136 DEBUG [Thread-1949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:06,136 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:43:06,216 INFO [Thread-1945 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1945. 2024-11-26T20:43:06,216 INFO [Thread-1943 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1943. 2024-11-26T20:43:06,216 DEBUG [Thread-1945 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:43:06,216 DEBUG [Thread-1943 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:43:06,216 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:06,216 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:06,216 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:06,216 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:06,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:43:06,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:43:06,225 INFO [Thread-1951 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1951. 2024-11-26T20:43:06,225 DEBUG [Thread-1951 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:43:06,226 DEBUG [Thread-1951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:06,226 DEBUG [Thread-1951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:06,226 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:43:06,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T20:43:06,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-26T20:43:06,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-26T20:43:06,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 352 2024-11-26T20:43:06,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-26T20:43:06,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-26T20:43:06,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T20:43:06,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T20:43:06,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 960 2024-11-26T20:43:06,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2867 rows 2024-11-26T20:43:06,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 949 2024-11-26T20:43:06,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2846 rows 2024-11-26T20:43:06,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 958 2024-11-26T20:43:06,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2869 rows 2024-11-26T20:43:06,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 954 2024-11-26T20:43:06,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2855 rows 2024-11-26T20:43:06,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 952 2024-11-26T20:43:06,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2855 rows 2024-11-26T20:43:06,226 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T20:43:06,226 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:43:06,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:06,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:06,226 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:43:06,227 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T20:43:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T20:43:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T20:43:06,229 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653786229"}]},"ts":"1732653786229"} 2024-11-26T20:43:06,230 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T20:43:06,230 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T20:43:06,231 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T20:43:06,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eaff56e8e250c06f0a9f8447be8c76bc, UNASSIGN}] 2024-11-26T20:43:06,233 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eaff56e8e250c06f0a9f8447be8c76bc, UNASSIGN 2024-11-26T20:43:06,233 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=eaff56e8e250c06f0a9f8447be8c76bc, regionState=CLOSING, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:43:06,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eaff56e8e250c06f0a9f8447be8c76bc, UNASSIGN because future has completed 2024-11-26T20:43:06,235 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T20:43:06,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795}] 2024-11-26T20:43:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T20:43:06,387 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:43:06,387 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T20:43:06,387 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing eaff56e8e250c06f0a9f8447be8c76bc, disabling compactions & flushes 2024-11-26T20:43:06,387 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:43:06,387 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:43:06,387 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. after waiting 0 ms 2024-11-26T20:43:06,387 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:43:06,387 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing eaff56e8e250c06f0a9f8447be8c76bc 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-26T20:43:06,387 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=A 2024-11-26T20:43:06,387 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:06,387 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=B 2024-11-26T20:43:06,387 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:06,387 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaff56e8e250c06f0a9f8447be8c76bc, store=C 2024-11-26T20:43:06,387 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:06,390 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/acb72147ed5640aba310149038f5e706 is 50, key is test_row_0/A:col10/1732653786224/Put/seqid=0 2024-11-26T20:43:06,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742340_1516 (size=7415) 2024-11-26T20:43:06,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T20:43:06,794 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/acb72147ed5640aba310149038f5e706 2024-11-26T20:43:06,799 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/91fe8574c17d409397f6a4f162aa242f is 50, key is test_row_0/B:col10/1732653786224/Put/seqid=0 2024-11-26T20:43:06,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742341_1517 (size=7415) 2024-11-26T20:43:06,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T20:43:07,203 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/91fe8574c17d409397f6a4f162aa242f 2024-11-26T20:43:07,208 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/250faf2bba674eeaa0a040e68d615a2e is 50, key is test_row_0/C:col10/1732653786224/Put/seqid=0 2024-11-26T20:43:07,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742342_1518 (size=7415) 2024-11-26T20:43:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T20:43:07,611 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/250faf2bba674eeaa0a040e68d615a2e 2024-11-26T20:43:07,614 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/A/acb72147ed5640aba310149038f5e706 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/acb72147ed5640aba310149038f5e706 2024-11-26T20:43:07,617 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/acb72147ed5640aba310149038f5e706, entries=50, sequenceid=588, filesize=7.2 K 2024-11-26T20:43:07,617 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/B/91fe8574c17d409397f6a4f162aa242f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/91fe8574c17d409397f6a4f162aa242f 2024-11-26T20:43:07,619 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/91fe8574c17d409397f6a4f162aa242f, entries=50, sequenceid=588, filesize=7.2 K 2024-11-26T20:43:07,620 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/.tmp/C/250faf2bba674eeaa0a040e68d615a2e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/250faf2bba674eeaa0a040e68d615a2e 2024-11-26T20:43:07,622 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/250faf2bba674eeaa0a040e68d615a2e, entries=50, sequenceid=588, filesize=7.2 K 2024-11-26T20:43:07,623 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for eaff56e8e250c06f0a9f8447be8c76bc in 1236ms, sequenceid=588, compaction requested=true 2024-11-26T20:43:07,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/d796fd99527d44bbbd2a2df801f74a05, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/a65d87126fce49d9b94e41c6ef96c8d3, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/9cf4ad50ad6c43449269b058c8a8908b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/9f36f5732d944b2e8cce1df77fa02242, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/ad5fbea536ac47fea18400c8ff9c5301, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/6539927b3f5b438daeb6e9c143bf1d65, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/8181856e3a32497bba4a5b31aad781af, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/fd4b4b47f9ad47249c84b56f5065720b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/e6e862b6073d4a7b9b80eb7fe7e249da, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/f2dc31b78dd54c3c8410f7e4f10cc763, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/65b4b179b03445748a41eb86905fc99a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/05f28ed83dd74527aecf15a32e1eb625, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/3e4308d7a91f4104b0ec8657eb51d146, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/879e3bff75a84d6d9aa571f65e7bd325, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/6cb87efb25bb44828c419636f289e76b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/e9a005c5007942d391d468370ca9f325, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/3d2f4e0d875e497f949870bfc204fbd5, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/826a57773d92480f9ab85537780593f4, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/6e1cfee08ec74e14b0cd0476486e508a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/4023b977194f4316a0cc1e3806a5a1bf, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/3c6316b9a783472e861820de2c95b1e3, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/e7531f3ab9034907aee4e8b9f92a1306, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/fb21f78f593c40b9b43c074076c37cf1, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/28523c20979845bfa53de81dfbdd034b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/d219d7e1eaa041dcaadc03da57804788, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/f72cd2a185554081b32ee669e42ee22c] to archive 2024-11-26T20:43:07,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T20:43:07,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/d796fd99527d44bbbd2a2df801f74a05 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/d796fd99527d44bbbd2a2df801f74a05 2024-11-26T20:43:07,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/a65d87126fce49d9b94e41c6ef96c8d3 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/a65d87126fce49d9b94e41c6ef96c8d3 2024-11-26T20:43:07,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/9cf4ad50ad6c43449269b058c8a8908b to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/9cf4ad50ad6c43449269b058c8a8908b 2024-11-26T20:43:07,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/9f36f5732d944b2e8cce1df77fa02242 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/9f36f5732d944b2e8cce1df77fa02242 2024-11-26T20:43:07,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/ad5fbea536ac47fea18400c8ff9c5301 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/ad5fbea536ac47fea18400c8ff9c5301 2024-11-26T20:43:07,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/6539927b3f5b438daeb6e9c143bf1d65 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/6539927b3f5b438daeb6e9c143bf1d65 2024-11-26T20:43:07,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/8181856e3a32497bba4a5b31aad781af to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/8181856e3a32497bba4a5b31aad781af 2024-11-26T20:43:07,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/fd4b4b47f9ad47249c84b56f5065720b to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/fd4b4b47f9ad47249c84b56f5065720b 2024-11-26T20:43:07,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/e6e862b6073d4a7b9b80eb7fe7e249da to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/e6e862b6073d4a7b9b80eb7fe7e249da 2024-11-26T20:43:07,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/f2dc31b78dd54c3c8410f7e4f10cc763 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/f2dc31b78dd54c3c8410f7e4f10cc763 2024-11-26T20:43:07,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/65b4b179b03445748a41eb86905fc99a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/65b4b179b03445748a41eb86905fc99a 2024-11-26T20:43:07,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/05f28ed83dd74527aecf15a32e1eb625 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/05f28ed83dd74527aecf15a32e1eb625 2024-11-26T20:43:07,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/3e4308d7a91f4104b0ec8657eb51d146 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/3e4308d7a91f4104b0ec8657eb51d146 2024-11-26T20:43:07,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/879e3bff75a84d6d9aa571f65e7bd325 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/879e3bff75a84d6d9aa571f65e7bd325 2024-11-26T20:43:07,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/6cb87efb25bb44828c419636f289e76b to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/6cb87efb25bb44828c419636f289e76b 2024-11-26T20:43:07,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/e9a005c5007942d391d468370ca9f325 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/e9a005c5007942d391d468370ca9f325 2024-11-26T20:43:07,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/3d2f4e0d875e497f949870bfc204fbd5 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/3d2f4e0d875e497f949870bfc204fbd5 2024-11-26T20:43:07,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/826a57773d92480f9ab85537780593f4 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/826a57773d92480f9ab85537780593f4 2024-11-26T20:43:07,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/6e1cfee08ec74e14b0cd0476486e508a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/6e1cfee08ec74e14b0cd0476486e508a 2024-11-26T20:43:07,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/4023b977194f4316a0cc1e3806a5a1bf to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/4023b977194f4316a0cc1e3806a5a1bf 2024-11-26T20:43:07,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/3c6316b9a783472e861820de2c95b1e3 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/3c6316b9a783472e861820de2c95b1e3 2024-11-26T20:43:07,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/e7531f3ab9034907aee4e8b9f92a1306 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/e7531f3ab9034907aee4e8b9f92a1306 2024-11-26T20:43:07,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/fb21f78f593c40b9b43c074076c37cf1 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/fb21f78f593c40b9b43c074076c37cf1 2024-11-26T20:43:07,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/28523c20979845bfa53de81dfbdd034b to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/28523c20979845bfa53de81dfbdd034b 2024-11-26T20:43:07,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/d219d7e1eaa041dcaadc03da57804788 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/d219d7e1eaa041dcaadc03da57804788 2024-11-26T20:43:07,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/f72cd2a185554081b32ee669e42ee22c to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/f72cd2a185554081b32ee669e42ee22c 2024-11-26T20:43:07,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/de848da6cebe456f8d3bd678eba05ca9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/d0642dde2dcd4bd68c24f251a36f65e8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/89b9738b4211436e95af3b106d9ada18, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/82bb843892704aa08e7147627e928551, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/ad73584dc5674626988af4bb18e7e005, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/0c37710946e6469a8f5f606f44d494a0, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/1605d4fd906a41049298b3c2a2d94e70, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/11c4fdc390aa463084d94a71b8a95929, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/504f047e05ac46708719136097211c80, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/fb7d754d39fd42079dc0af0bbe553e37, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/8b76bbf61b254f21b21f673b598359a7, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/fd832a83f49449a89fcf42c347e1d686, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/83f70ac2bbeb44e28787907b2ee8dd27, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/fcdf6b263a8b49d8abd33431efeb1554, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/0645a12e57c14015ab356c0383f3141a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/4dd6ddd6a023414993055d8883adb9b5, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/06a80908b46e4d57a20ccc441b0ced01, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/d7d2b31340204e049f6e4fa8541b979b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/60effdacd5724fcab1035a3d3b2b3442, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/3eb88d44c9d945238ddf45393652de11, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/d5aaa96b2cda4680809ca0be7ac9f267, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/79d79d800e444d5a9d286304ebc73a70, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/fa5385f70797449899fa67ea0353023b, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/34953f64a33543d5bcee919ae2baaff5, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/909487a9d8d846bfa3b195c3fa38778a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/c738ffc196e94767a80629309ca7bc6b] to archive 2024-11-26T20:43:07,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T20:43:07,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/de848da6cebe456f8d3bd678eba05ca9 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/de848da6cebe456f8d3bd678eba05ca9 2024-11-26T20:43:07,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/d0642dde2dcd4bd68c24f251a36f65e8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/d0642dde2dcd4bd68c24f251a36f65e8 2024-11-26T20:43:07,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/89b9738b4211436e95af3b106d9ada18 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/89b9738b4211436e95af3b106d9ada18 2024-11-26T20:43:07,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/82bb843892704aa08e7147627e928551 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/82bb843892704aa08e7147627e928551 2024-11-26T20:43:07,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/ad73584dc5674626988af4bb18e7e005 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/ad73584dc5674626988af4bb18e7e005 2024-11-26T20:43:07,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/0c37710946e6469a8f5f606f44d494a0 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/0c37710946e6469a8f5f606f44d494a0 2024-11-26T20:43:07,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/1605d4fd906a41049298b3c2a2d94e70 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/1605d4fd906a41049298b3c2a2d94e70 2024-11-26T20:43:07,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/11c4fdc390aa463084d94a71b8a95929 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/11c4fdc390aa463084d94a71b8a95929 2024-11-26T20:43:07,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/504f047e05ac46708719136097211c80 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/504f047e05ac46708719136097211c80 2024-11-26T20:43:07,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/fb7d754d39fd42079dc0af0bbe553e37 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/fb7d754d39fd42079dc0af0bbe553e37 2024-11-26T20:43:07,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/8b76bbf61b254f21b21f673b598359a7 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/8b76bbf61b254f21b21f673b598359a7 2024-11-26T20:43:07,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/fd832a83f49449a89fcf42c347e1d686 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/fd832a83f49449a89fcf42c347e1d686 2024-11-26T20:43:07,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/83f70ac2bbeb44e28787907b2ee8dd27 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/83f70ac2bbeb44e28787907b2ee8dd27 2024-11-26T20:43:07,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/fcdf6b263a8b49d8abd33431efeb1554 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/fcdf6b263a8b49d8abd33431efeb1554 2024-11-26T20:43:07,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/0645a12e57c14015ab356c0383f3141a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/0645a12e57c14015ab356c0383f3141a 2024-11-26T20:43:07,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/4dd6ddd6a023414993055d8883adb9b5 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/4dd6ddd6a023414993055d8883adb9b5 2024-11-26T20:43:07,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/06a80908b46e4d57a20ccc441b0ced01 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/06a80908b46e4d57a20ccc441b0ced01 2024-11-26T20:43:07,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/d7d2b31340204e049f6e4fa8541b979b to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/d7d2b31340204e049f6e4fa8541b979b 2024-11-26T20:43:07,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/60effdacd5724fcab1035a3d3b2b3442 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/60effdacd5724fcab1035a3d3b2b3442 2024-11-26T20:43:07,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/3eb88d44c9d945238ddf45393652de11 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/3eb88d44c9d945238ddf45393652de11 2024-11-26T20:43:07,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/d5aaa96b2cda4680809ca0be7ac9f267 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/d5aaa96b2cda4680809ca0be7ac9f267 2024-11-26T20:43:07,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/79d79d800e444d5a9d286304ebc73a70 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/79d79d800e444d5a9d286304ebc73a70 2024-11-26T20:43:07,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/fa5385f70797449899fa67ea0353023b to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/fa5385f70797449899fa67ea0353023b 2024-11-26T20:43:07,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/34953f64a33543d5bcee919ae2baaff5 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/34953f64a33543d5bcee919ae2baaff5 2024-11-26T20:43:07,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/909487a9d8d846bfa3b195c3fa38778a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/909487a9d8d846bfa3b195c3fa38778a 2024-11-26T20:43:07,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/c738ffc196e94767a80629309ca7bc6b to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/c738ffc196e94767a80629309ca7bc6b 2024-11-26T20:43:07,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/8ce1525c74f4430aae3bb20586689c2c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/c1deaef79e6e42cd81e4e37972646494, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/e88e7b73a3cb40aebba784acd71a0976, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0b15e94c3c0947b39675ddfa056ffc17, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0ddcc400f5d8463e886cc642c04494fd, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/c23c51181798404ca0bfeb381f24a576, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/bc2be58dd28b42b0b2de8c7a098fe982, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/318f8dbc6ab5436eb0f4e901a59612c8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/8399bf6d73454518b0c07dca7854b72a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/effe8c71e5d24629a58052a41d47d018, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/d6b57f30c1cb40608fee638aeb5ca7f5, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/60240ab7c3a846429f8fd1a42ac29bbf, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/6f23d9ec05824faba6af3dddf8c1fe3a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/e2e26842bb4c44d0baffc1294638bcc8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/e8bc74178e404fc3838ae7d718b2c6e0, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/6cc5f55302d04de98c5aafdf65748b38, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/353cd00243cb4a7ba28d9c14e36a406e, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/2a5c65d43f824f8fb154b04ad2f3c6c2, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/710381c9b00e4ebe8762274c46eb5116, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/93914cd79fc74bb882fa0bf715f48016, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0a02eecded57405cbb4cd56c2f5f986e, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0df1c81d93c84b05899470d62e2d584c, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/7a8784eb382b492ca7ecca0727c6512a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/4eeac7bfaf6146a2be172bde3ef65a24, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/57abf73d0b9e42e18719d8ee1b5b8afe, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/22ff9951166d4d37b2dd53737aff2c71] to archive 2024-11-26T20:43:07,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T20:43:07,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/8ce1525c74f4430aae3bb20586689c2c to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/8ce1525c74f4430aae3bb20586689c2c 2024-11-26T20:43:07,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/c1deaef79e6e42cd81e4e37972646494 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/c1deaef79e6e42cd81e4e37972646494 2024-11-26T20:43:07,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/e88e7b73a3cb40aebba784acd71a0976 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/e88e7b73a3cb40aebba784acd71a0976 2024-11-26T20:43:07,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0b15e94c3c0947b39675ddfa056ffc17 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0b15e94c3c0947b39675ddfa056ffc17 2024-11-26T20:43:07,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0ddcc400f5d8463e886cc642c04494fd to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0ddcc400f5d8463e886cc642c04494fd 2024-11-26T20:43:07,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/c23c51181798404ca0bfeb381f24a576 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/c23c51181798404ca0bfeb381f24a576 2024-11-26T20:43:07,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/bc2be58dd28b42b0b2de8c7a098fe982 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/bc2be58dd28b42b0b2de8c7a098fe982 2024-11-26T20:43:07,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/318f8dbc6ab5436eb0f4e901a59612c8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/318f8dbc6ab5436eb0f4e901a59612c8 2024-11-26T20:43:07,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/8399bf6d73454518b0c07dca7854b72a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/8399bf6d73454518b0c07dca7854b72a 2024-11-26T20:43:07,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/effe8c71e5d24629a58052a41d47d018 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/effe8c71e5d24629a58052a41d47d018 2024-11-26T20:43:07,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/d6b57f30c1cb40608fee638aeb5ca7f5 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/d6b57f30c1cb40608fee638aeb5ca7f5 2024-11-26T20:43:07,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/60240ab7c3a846429f8fd1a42ac29bbf to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/60240ab7c3a846429f8fd1a42ac29bbf 2024-11-26T20:43:07,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/6f23d9ec05824faba6af3dddf8c1fe3a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/6f23d9ec05824faba6af3dddf8c1fe3a 2024-11-26T20:43:07,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/e2e26842bb4c44d0baffc1294638bcc8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/e2e26842bb4c44d0baffc1294638bcc8 2024-11-26T20:43:07,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/e8bc74178e404fc3838ae7d718b2c6e0 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/e8bc74178e404fc3838ae7d718b2c6e0 2024-11-26T20:43:07,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/6cc5f55302d04de98c5aafdf65748b38 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/6cc5f55302d04de98c5aafdf65748b38 2024-11-26T20:43:07,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/353cd00243cb4a7ba28d9c14e36a406e to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/353cd00243cb4a7ba28d9c14e36a406e 2024-11-26T20:43:07,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/2a5c65d43f824f8fb154b04ad2f3c6c2 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/2a5c65d43f824f8fb154b04ad2f3c6c2 2024-11-26T20:43:07,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/710381c9b00e4ebe8762274c46eb5116 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/710381c9b00e4ebe8762274c46eb5116 2024-11-26T20:43:07,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/93914cd79fc74bb882fa0bf715f48016 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/93914cd79fc74bb882fa0bf715f48016 2024-11-26T20:43:07,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0a02eecded57405cbb4cd56c2f5f986e to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0a02eecded57405cbb4cd56c2f5f986e 2024-11-26T20:43:07,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0df1c81d93c84b05899470d62e2d584c to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/0df1c81d93c84b05899470d62e2d584c 2024-11-26T20:43:07,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/7a8784eb382b492ca7ecca0727c6512a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/7a8784eb382b492ca7ecca0727c6512a 2024-11-26T20:43:07,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/4eeac7bfaf6146a2be172bde3ef65a24 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/4eeac7bfaf6146a2be172bde3ef65a24 2024-11-26T20:43:07,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/57abf73d0b9e42e18719d8ee1b5b8afe to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/57abf73d0b9e42e18719d8ee1b5b8afe 2024-11-26T20:43:07,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/22ff9951166d4d37b2dd53737aff2c71 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/22ff9951166d4d37b2dd53737aff2c71 2024-11-26T20:43:07,690 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/recovered.edits/591.seqid, newMaxSeqId=591, maxSeqId=1 2024-11-26T20:43:07,691 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. 2024-11-26T20:43:07,691 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for eaff56e8e250c06f0a9f8447be8c76bc: Waiting for close lock at 1732653786387Running coprocessor pre-close hooks at 1732653786387Disabling compacts and flushes for region at 1732653786387Disabling writes for close at 1732653786387Obtaining lock to block concurrent updates at 1732653786387Preparing flush snapshotting stores in eaff56e8e250c06f0a9f8447be8c76bc at 1732653786387Finished memstore snapshotting TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732653786387Flushing stores of TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc. at 1732653786388 (+1 ms)Flushing eaff56e8e250c06f0a9f8447be8c76bc/A: creating writer at 1732653786388Flushing eaff56e8e250c06f0a9f8447be8c76bc/A: appending metadata at 1732653786390 (+2 ms)Flushing eaff56e8e250c06f0a9f8447be8c76bc/A: closing flushed file at 1732653786390Flushing eaff56e8e250c06f0a9f8447be8c76bc/B: creating writer at 1732653786797 (+407 ms)Flushing eaff56e8e250c06f0a9f8447be8c76bc/B: appending metadata at 1732653786799 (+2 ms)Flushing eaff56e8e250c06f0a9f8447be8c76bc/B: closing flushed file at 1732653786799Flushing eaff56e8e250c06f0a9f8447be8c76bc/C: creating writer at 1732653787205 (+406 ms)Flushing eaff56e8e250c06f0a9f8447be8c76bc/C: appending metadata at 1732653787207 (+2 ms)Flushing eaff56e8e250c06f0a9f8447be8c76bc/C: closing flushed file at 1732653787207Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74a6bfae: reopening flushed file at 1732653787614 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9f2fa69: reopening flushed file at 1732653787617 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@399ce496: reopening flushed file at 1732653787620 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for eaff56e8e250c06f0a9f8447be8c76bc in 1236ms, sequenceid=588, compaction requested=true at 1732653787623 (+3 ms)Writing region close event to WAL at 1732653787688 (+65 ms)Running coprocessor post-close hooks at 1732653787691 (+3 ms)Closed at 1732653787691 2024-11-26T20:43:07,692 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:43:07,693 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=eaff56e8e250c06f0a9f8447be8c76bc, regionState=CLOSED 2024-11-26T20:43:07,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 because future has completed 2024-11-26T20:43:07,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-26T20:43:07,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure eaff56e8e250c06f0a9f8447be8c76bc, server=cd6be161543e,40291,1732653631795 in 1.4600 sec 2024-11-26T20:43:07,697 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-26T20:43:07,697 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eaff56e8e250c06f0a9f8447be8c76bc, UNASSIGN in 1.4640 sec 2024-11-26T20:43:07,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-26T20:43:07,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4670 sec 2024-11-26T20:43:07,700 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653787700"}]},"ts":"1732653787700"} 2024-11-26T20:43:07,701 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T20:43:07,701 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T20:43:07,702 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4740 sec 2024-11-26T20:43:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T20:43:08,384 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:43:08,384 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T20:43:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:43:08,386 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:43:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-26T20:43:08,386 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:43:08,388 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:43:08,390 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A, FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B, FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C, FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/recovered.edits] 2024-11-26T20:43:08,391 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/28469b7f5e824249ae0849d7d6e40f64 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/28469b7f5e824249ae0849d7d6e40f64 2024-11-26T20:43:08,392 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/50eee35ee8a8455b94c85b0fa1b1774b to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/50eee35ee8a8455b94c85b0fa1b1774b 2024-11-26T20:43:08,393 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/acb72147ed5640aba310149038f5e706 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/A/acb72147ed5640aba310149038f5e706 2024-11-26T20:43:08,395 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/91fe8574c17d409397f6a4f162aa242f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/91fe8574c17d409397f6a4f162aa242f 2024-11-26T20:43:08,396 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/b2c8b609411f4947bf4e8371898fc907 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/b2c8b609411f4947bf4e8371898fc907 2024-11-26T20:43:08,397 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/c17033d408074c938934c47bdfaf4731 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/B/c17033d408074c938934c47bdfaf4731 2024-11-26T20:43:08,398 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/250faf2bba674eeaa0a040e68d615a2e to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/250faf2bba674eeaa0a040e68d615a2e 2024-11-26T20:43:08,399 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/bcaf7e9422ad4be699bd9617b4277aef to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/bcaf7e9422ad4be699bd9617b4277aef 2024-11-26T20:43:08,400 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/c3dd4d153414495cbd09a114aa74d37d to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/C/c3dd4d153414495cbd09a114aa74d37d 2024-11-26T20:43:08,402 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/recovered.edits/591.seqid to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc/recovered.edits/591.seqid 2024-11-26T20:43:08,402 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/eaff56e8e250c06f0a9f8447be8c76bc 2024-11-26T20:43:08,402 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T20:43:08,404 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:43:08,405 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T20:43:08,407 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T20:43:08,408 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:43:08,408 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T20:43:08,408 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732653788408"}]},"ts":"9223372036854775807"} 2024-11-26T20:43:08,409 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T20:43:08,409 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => eaff56e8e250c06f0a9f8447be8c76bc, NAME => 'TestAcidGuarantees,,1732653757072.eaff56e8e250c06f0a9f8447be8c76bc.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T20:43:08,409 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T20:43:08,410 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732653788409"}]},"ts":"9223372036854775807"} 2024-11-26T20:43:08,411 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T20:43:08,411 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:43:08,412 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 27 msec 2024-11-26T20:43:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-26T20:43:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T20:43:08,494 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:43:08,503 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248 (was 252), OpenFileDescriptor=447 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=400 (was 380) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5147 (was 5243) 2024-11-26T20:43:08,511 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=400, ProcessCount=11, AvailableMemoryMB=5147 2024-11-26T20:43:08,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T20:43:08,512 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T20:43:08,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T20:43:08,514 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T20:43:08,514 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:08,514 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-11-26T20:43:08,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-26T20:43:08,515 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T20:43:08,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742343_1519 (size=1061) 2024-11-26T20:43:08,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-26T20:43:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-26T20:43:08,921 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794 2024-11-26T20:43:08,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742344_1520 (size=53) 2024-11-26T20:43:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-26T20:43:09,325 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:43:09,325 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0c644311c9706b1426d8809c603e0fe8, disabling compactions & flushes 2024-11-26T20:43:09,325 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:09,325 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:09,325 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. after waiting 0 ms 2024-11-26T20:43:09,325 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:09,325 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:09,325 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0c644311c9706b1426d8809c603e0fe8: Waiting for close lock at 1732653789325Disabling compacts and flushes for region at 1732653789325Disabling writes for close at 1732653789325Writing region close event to WAL at 1732653789325Closed at 1732653789325 2024-11-26T20:43:09,326 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T20:43:09,327 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732653789326"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732653789326"}]},"ts":"1732653789326"} 2024-11-26T20:43:09,328 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T20:43:09,329 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T20:43:09,329 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653789329"}]},"ts":"1732653789329"} 2024-11-26T20:43:09,330 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T20:43:09,330 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0c644311c9706b1426d8809c603e0fe8, ASSIGN}] 2024-11-26T20:43:09,331 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0c644311c9706b1426d8809c603e0fe8, ASSIGN 2024-11-26T20:43:09,331 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0c644311c9706b1426d8809c603e0fe8, ASSIGN; state=OFFLINE, location=cd6be161543e,40291,1732653631795; forceNewPlan=false, retain=false 2024-11-26T20:43:09,482 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=0c644311c9706b1426d8809c603e0fe8, regionState=OPENING, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:43:09,483 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0c644311c9706b1426d8809c603e0fe8, ASSIGN because future has completed 2024-11-26T20:43:09,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795}] 2024-11-26T20:43:09,637 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:09,637 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:43:09,638 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:09,638 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:43:09,638 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:09,638 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:09,639 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:09,640 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:43:09,640 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c644311c9706b1426d8809c603e0fe8 columnFamilyName A 2024-11-26T20:43:09,640 DEBUG [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:09,640 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] regionserver.HStore(327): Store=0c644311c9706b1426d8809c603e0fe8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:43:09,640 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:09,641 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:43:09,641 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c644311c9706b1426d8809c603e0fe8 columnFamilyName B 2024-11-26T20:43:09,641 DEBUG [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:09,641 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] regionserver.HStore(327): Store=0c644311c9706b1426d8809c603e0fe8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:43:09,641 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:09,642 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:43:09,642 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c644311c9706b1426d8809c603e0fe8 columnFamilyName C 2024-11-26T20:43:09,642 DEBUG [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:09,642 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] regionserver.HStore(327): Store=0c644311c9706b1426d8809c603e0fe8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:43:09,643 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:09,643 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:09,643 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:09,643 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:09,644 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:09,644 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:09,644 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T20:43:09,645 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:09,647 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:43:09,647 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened 0c644311c9706b1426d8809c603e0fe8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68924362, jitterRate=0.027053028345108032}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T20:43:09,647 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:09,647 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for 0c644311c9706b1426d8809c603e0fe8: Running coprocessor pre-open hook at 1732653789638Writing region info on filesystem at 1732653789638Initializing all the Stores at 1732653789638Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653789638Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653789639 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653789639Cleaning up temporary data from old regions at 1732653789644 (+5 ms)Running coprocessor post-open hooks at 1732653789647 (+3 ms)Region opened successfully at 1732653789647 2024-11-26T20:43:09,648 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., pid=140, masterSystemTime=1732653789635 2024-11-26T20:43:09,649 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:09,649 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:09,650 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=0c644311c9706b1426d8809c603e0fe8, regionState=OPEN, openSeqNum=2, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:43:09,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 because future has completed 2024-11-26T20:43:09,652 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-26T20:43:09,652 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 in 168 msec 2024-11-26T20:43:09,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-26T20:43:09,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0c644311c9706b1426d8809c603e0fe8, ASSIGN in 322 msec 2024-11-26T20:43:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-26T20:43:09,654 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T20:43:09,654 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653789654"}]},"ts":"1732653789654"} 2024-11-26T20:43:09,655 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T20:43:09,656 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T20:43:09,657 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1440 sec 2024-11-26T20:43:10,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-26T20:43:10,674 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:43:10,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d463f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:10,675 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:43:10,675 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:43:10,676 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:43:10,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:43:10,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:43:10,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3117b36c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:10,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:43:10,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:43:10,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:10,677 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:43:10,678 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7daeac1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:10,678 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:43:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:43:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:43:10,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:43:10,680 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T20:43:10,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is cd6be161543e,42359,1732653630959 2024-11-26T20:43:10,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5010c991 2024-11-26T20:43:10,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T20:43:10,682 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T20:43:10,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T20:43:10,683 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T20:43:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-26T20:43:10,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742345_1521 (size=1097) 2024-11-26T20:43:11,093 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-26T20:43:11,093 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-26T20:43:11,095 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T20:43:11,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0c644311c9706b1426d8809c603e0fe8, REOPEN/MOVE}] 2024-11-26T20:43:11,097 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0c644311c9706b1426d8809c603e0fe8, REOPEN/MOVE 2024-11-26T20:43:11,098 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=0c644311c9706b1426d8809c603e0fe8, regionState=CLOSING, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:43:11,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0c644311c9706b1426d8809c603e0fe8, REOPEN/MOVE because future has completed 2024-11-26T20:43:11,099 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T20:43:11,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795}] 2024-11-26T20:43:11,251 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:11,251 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T20:43:11,252 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 0c644311c9706b1426d8809c603e0fe8, disabling compactions & flushes 2024-11-26T20:43:11,252 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:11,252 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:11,252 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. after waiting 0 ms 2024-11-26T20:43:11,252 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:11,255 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:43:11,255 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:11,255 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 0c644311c9706b1426d8809c603e0fe8: Waiting for close lock at 1732653791252Running coprocessor pre-close hooks at 1732653791252Disabling compacts and flushes for region at 1732653791252Disabling writes for close at 1732653791252Writing region close event to WAL at 1732653791252Running coprocessor post-close hooks at 1732653791255 (+3 ms)Closed at 1732653791255 2024-11-26T20:43:11,255 WARN [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: 0c644311c9706b1426d8809c603e0fe8 to self. 2024-11-26T20:43:11,257 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:11,257 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=0c644311c9706b1426d8809c603e0fe8, regionState=CLOSED 2024-11-26T20:43:11,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 because future has completed 2024-11-26T20:43:11,260 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-26T20:43:11,260 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 in 160 msec 2024-11-26T20:43:11,261 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0c644311c9706b1426d8809c603e0fe8, REOPEN/MOVE; state=CLOSED, location=cd6be161543e,40291,1732653631795; forceNewPlan=false, retain=true 2024-11-26T20:43:11,411 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=0c644311c9706b1426d8809c603e0fe8, regionState=OPENING, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:43:11,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0c644311c9706b1426d8809c603e0fe8, REOPEN/MOVE because future has completed 2024-11-26T20:43:11,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795}] 2024-11-26T20:43:11,414 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-26T20:43:11,566 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:11,566 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:43:11,567 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:11,567 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:43:11,567 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:11,567 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:11,568 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:11,568 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:43:11,569 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c644311c9706b1426d8809c603e0fe8 columnFamilyName A 2024-11-26T20:43:11,570 DEBUG [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:11,570 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] regionserver.HStore(327): Store=0c644311c9706b1426d8809c603e0fe8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:43:11,570 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:11,571 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:43:11,571 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c644311c9706b1426d8809c603e0fe8 columnFamilyName B 2024-11-26T20:43:11,571 DEBUG [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:11,571 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] regionserver.HStore(327): Store=0c644311c9706b1426d8809c603e0fe8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:43:11,571 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:11,572 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T20:43:11,572 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c644311c9706b1426d8809c603e0fe8 columnFamilyName C 2024-11-26T20:43:11,572 DEBUG [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:11,572 INFO [StoreOpener-0c644311c9706b1426d8809c603e0fe8-1 {}] regionserver.HStore(327): Store=0c644311c9706b1426d8809c603e0fe8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:43:11,572 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:11,572 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:11,573 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:11,573 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:11,574 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:11,574 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:11,574 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T20:43:11,575 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:11,575 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened 0c644311c9706b1426d8809c603e0fe8; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73406958, jitterRate=0.09384891390800476}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T20:43:11,576 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:11,576 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for 0c644311c9706b1426d8809c603e0fe8: Running coprocessor pre-open hook at 1732653791567Writing region info on filesystem at 1732653791567Initializing all the Stores at 1732653791567Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653791568 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653791568Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732653791568Cleaning up temporary data from old regions at 1732653791574 (+6 ms)Running coprocessor post-open hooks at 1732653791576 (+2 ms)Region opened successfully at 1732653791576 2024-11-26T20:43:11,577 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., pid=145, masterSystemTime=1732653791564 2024-11-26T20:43:11,578 DEBUG [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:11,578 INFO [RS_OPEN_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:11,579 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=0c644311c9706b1426d8809c603e0fe8, regionState=OPEN, openSeqNum=5, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:43:11,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 because future has completed 2024-11-26T20:43:11,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-11-26T20:43:11,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 in 167 msec 2024-11-26T20:43:11,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-26T20:43:11,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0c644311c9706b1426d8809c603e0fe8, REOPEN/MOVE in 485 msec 2024-11-26T20:43:11,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-26T20:43:11,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 488 msec 2024-11-26T20:43:11,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 903 msec 2024-11-26T20:43:11,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-26T20:43:11,587 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:43:11,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26378a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:43:11,588 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:43:11,589 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:43:11,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:43:11,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:43:11,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29506308, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:43:11,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:43:11,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:11,590 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:43:11,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17c00de6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,591 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:43:11,591 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:43:11,592 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:43:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:43:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:43:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26694ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:43:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:43:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:11,593 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:43:11,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20ed4e78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,594 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:43:11,594 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:43:11,594 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:43:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:43:11,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:43:11,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@522a0311, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:43:11,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:43:11,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:11,595 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:43:11,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@674a3743, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,596 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:43:11,596 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:43:11,597 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:43:11,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:43:11,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:43:11,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ab681d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:43:11,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:43:11,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:11,598 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:43:11,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@327124c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,599 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,599 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:43:11,599 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:43:11,599 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:43:11,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:43:11,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:43:11,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47260532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:43:11,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:43:11,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:11,601 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:43:11,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15b5fb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,601 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,601 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:43:11,601 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:43:11,602 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:43:11,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:43:11,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:43:11,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15332d3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:43:11,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:43:11,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:11,603 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:43:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72630bc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,604 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:43:11,604 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:43:11,604 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:43:11,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:43:11,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:43:11,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5bb66f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:43:11,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:43:11,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:11,606 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:43:11,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b304fdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,606 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:43:11,606 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:43:11,607 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:43:11,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:43:11,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:43:11,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e04bf9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:43:11,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:43:11,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:11,608 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:43:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d40f55f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,609 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:43:11,609 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:43:11,610 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:43:11,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:43:11,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:43:11,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@329c8fad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:43:11,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:43:11,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:11,611 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:43:11,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25be093f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,612 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cd6be161543e,42359,-1 for getting cluster id 2024-11-26T20:43:11,612 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:43:11,612 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'de4c8973-112a-42f4-a492-8da7ceac9c1e' 2024-11-26T20:43:11,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:43:11,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "de4c8973-112a-42f4-a492-8da7ceac9c1e" 2024-11-26T20:43:11,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c855a79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cd6be161543e,42359,-1] 2024-11-26T20:43:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:43:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:11,613 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:43:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6de942b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:43:11,614 DEBUG [Thread-2341 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:43:11,614 DEBUG [Thread-2333 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:43:11,615 DEBUG [Thread-2339 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:43:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:43:11,615 DEBUG [Thread-2352 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:43:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:43:11,615 DEBUG [Thread-2335 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:43:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:43:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:43:11,615 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:43:11,615 DEBUG [Thread-2337 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:43:11,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:43:11,616 DEBUG [Thread-2348 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:43:11,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:43:11,616 DEBUG [Thread-2346 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:43:11,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:43:11,616 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:43:11,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:43:11,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:43:11,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:43:11,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:43:11,617 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:43:11,617 DEBUG [Thread-2348 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:43:11,617 DEBUG [Thread-2348 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:43:11,617 DEBUG [Thread-2346 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:43:11,617 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:43:11,617 DEBUG [Thread-2346 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:43:11,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-26T20:43:11,617 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:43:11,617 DEBUG [Thread-2335 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:43:11,617 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:43:11,618 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:43:11,618 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46192, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:43:11,618 DEBUG [Thread-2344 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:43:11,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T20:43:11,618 DEBUG [Thread-2350 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:43:11,618 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:43:11,619 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:43:11,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:43:11,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:43:11,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cd6be161543e,40291,1732653631795, seqNum=-1] 2024-11-26T20:43:11,619 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:43:11,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:43:11,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:43:11,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:43:11,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:43:11,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:43:11,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:43:11,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:43:11,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:43:11,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:43:11,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:43:11,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:43:11,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:43:11,624 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:43:11,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5] 2024-11-26T20:43:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:11,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0c644311c9706b1426d8809c603e0fe8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T20:43:11,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=A 2024-11-26T20:43:11,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:11,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=B 2024-11-26T20:43:11,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:11,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=C 2024-11-26T20:43:11,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:11,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653791648, max timestamp=1732653791661Num uniques 859; 2024-11-26T20:43:11,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653791648, max timestamp=1732653791661Num uniques 859; 2024-11-26T20:43:11,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:43:11,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:43:11,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112611c93f90a3d24093975bdee8f0278902_0c644311c9706b1426d8809c603e0fe8 is 50, key is test_row_0/A:col10/1732653791629/Put/seqid=0 2024-11-26T20:43:11,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653791648, max timestamp=1732653791661Num uniques 859; 2024-11-26T20:43:11,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:43:11,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:11,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653851678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:11,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:11,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:11,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:11,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:11,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653851680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:11,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:11,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:11,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1732653851681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:11,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:11,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:11,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:11,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653851682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:11,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:11,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:11,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:11,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:11,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:11,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:11,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:11,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653851684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:11,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:11,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:11,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:11,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742346_1522 (size=12154) 2024-11-26T20:43:11,697 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:11,704 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112611c93f90a3d24093975bdee8f0278902_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112611c93f90a3d24093975bdee8f0278902_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:11,704 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/17aac43a6e224bd2b960c6b48edf9ff9, store: [table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:11,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/17aac43a6e224bd2b960c6b48edf9ff9 is 175, key is test_row_0/A:col10/1732653791629/Put/seqid=0 2024-11-26T20:43:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T20:43:11,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742347_1523 (size=30955) 2024-11-26T20:43:11,743 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/17aac43a6e224bd2b960c6b48edf9ff9 2024-11-26T20:43:11,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-26T20:43:11,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:11,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:11,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:11,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:11,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:11,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/bc1d0778308d49659bfc371e4a956c5e is 50, key is test_row_0/B:col10/1732653791629/Put/seqid=0 2024-11-26T20:43:11,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653851795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:11,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1732653851795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:11,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653851795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:11,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653851795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:11,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653851795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:11,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:11,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:11,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:11,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:11,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:11,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:11,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742348_1524 (size=12001) 2024-11-26T20:43:11,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/bc1d0778308d49659bfc371e4a956c5e 2024-11-26T20:43:11,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/ae1754899b0f41169026ae72cda10d84 is 50, key is test_row_0/C:col10/1732653791629/Put/seqid=0 2024-11-26T20:43:11,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742349_1525 (size=12001) 2024-11-26T20:43:11,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/ae1754899b0f41169026ae72cda10d84 2024-11-26T20:43:11,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/17aac43a6e224bd2b960c6b48edf9ff9 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/17aac43a6e224bd2b960c6b48edf9ff9 2024-11-26T20:43:11,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/17aac43a6e224bd2b960c6b48edf9ff9, entries=150, sequenceid=16, filesize=30.2 K 2024-11-26T20:43:11,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/bc1d0778308d49659bfc371e4a956c5e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/bc1d0778308d49659bfc371e4a956c5e 2024-11-26T20:43:11,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/bc1d0778308d49659bfc371e4a956c5e, entries=150, sequenceid=16, filesize=11.7 K 2024-11-26T20:43:11,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/ae1754899b0f41169026ae72cda10d84 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/ae1754899b0f41169026ae72cda10d84 2024-11-26T20:43:11,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/ae1754899b0f41169026ae72cda10d84, entries=150, sequenceid=16, filesize=11.7 K 2024-11-26T20:43:11,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 0c644311c9706b1426d8809c603e0fe8 in 226ms, sequenceid=16, compaction requested=false 2024-11-26T20:43:11,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:11,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-26T20:43:11,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:11,928 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 0c644311c9706b1426d8809c603e0fe8 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-26T20:43:11,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=A 2024-11-26T20:43:11,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:43:11,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=B 2024-11-26T20:43:11,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:43:11,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=C 2024-11-26T20:43:11,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:43:11,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d2125163354943948db0c5294c1d39cb_0c644311c9706b1426d8809c603e0fe8 is 50, key is test_row_0/A:col10/1732653791680/Put/seqid=0 2024-11-26T20:43:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T20:43:11,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742350_1526 (size=12154) 2024-11-26T20:43:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:12,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:12,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1732653852016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:12,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:12,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653852017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653852017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653852018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653852018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:12,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1732653852124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:12,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:12,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653852134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:12,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:12,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653852134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653852134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653852134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:12,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T20:43:12,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1732653852334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:12,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:12,342 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d2125163354943948db0c5294c1d39cb_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d2125163354943948db0c5294c1d39cb_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:12,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/bee9024d6d014948b6b583859ecca752, store: [table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:12,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/bee9024d6d014948b6b583859ecca752 is 175, key is test_row_0/A:col10/1732653791680/Put/seqid=0 2024-11-26T20:43:12,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742351_1527 (size=30955) 2024-11-26T20:43:12,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653852344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:12,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:12,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653852344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:12,348 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/bee9024d6d014948b6b583859ecca752 2024-11-26T20:43:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:12,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653852344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:12,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653852344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:12,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/ab6460139e8344dd8d0e43f0ef4f35b8 is 50, key is test_row_0/B:col10/1732653791680/Put/seqid=0 2024-11-26T20:43:12,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742352_1528 (size=12001) 2024-11-26T20:43:12,365 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/ab6460139e8344dd8d0e43f0ef4f35b8 2024-11-26T20:43:12,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/fc0130d442304d87b84f419db3efa305 is 50, key is test_row_0/C:col10/1732653791680/Put/seqid=0 2024-11-26T20:43:12,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742353_1529 (size=12001) 2024-11-26T20:43:12,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1732653852643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:12,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:12,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653852654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:12,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:12,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653852654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:12,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:12,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:12,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653852654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:12,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653852654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T20:43:12,775 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/fc0130d442304d87b84f419db3efa305 2024-11-26T20:43:12,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/bee9024d6d014948b6b583859ecca752 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/bee9024d6d014948b6b583859ecca752 2024-11-26T20:43:12,782 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/bee9024d6d014948b6b583859ecca752, entries=150, sequenceid=50, filesize=30.2 K 2024-11-26T20:43:12,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/ab6460139e8344dd8d0e43f0ef4f35b8 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/ab6460139e8344dd8d0e43f0ef4f35b8 2024-11-26T20:43:12,786 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/ab6460139e8344dd8d0e43f0ef4f35b8, entries=150, sequenceid=50, filesize=11.7 K 2024-11-26T20:43:12,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/fc0130d442304d87b84f419db3efa305 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/fc0130d442304d87b84f419db3efa305 2024-11-26T20:43:12,790 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/fc0130d442304d87b84f419db3efa305, entries=150, sequenceid=50, filesize=11.7 K 2024-11-26T20:43:12,791 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=67.09 KB/68700 for 0c644311c9706b1426d8809c603e0fe8 in 863ms, sequenceid=50, compaction requested=false 2024-11-26T20:43:12,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:12,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:12,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-26T20:43:12,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-26T20:43:12,795 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-26T20:43:12,795 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1740 sec 2024-11-26T20:43:12,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.1810 sec 2024-11-26T20:43:13,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:13,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0c644311c9706b1426d8809c603e0fe8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T20:43:13,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=A 2024-11-26T20:43:13,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:13,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=B 2024-11-26T20:43:13,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:13,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=C 2024-11-26T20:43:13,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:13,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126834851ebf8be4a11b77bdf646df7fc6c_0c644311c9706b1426d8809c603e0fe8 is 50, key is test_row_0/A:col10/1732653792015/Put/seqid=0 2024-11-26T20:43:13,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742354_1530 (size=14594) 2024-11-26T20:43:13,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:13,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653853182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:13,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:13,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:13,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653853188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:13,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:13,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1732653853188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:13,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653853189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:13,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:13,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653853193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:13,211 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T20:43:13,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:13,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653853294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:13,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:13,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:13,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1732653853304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:13,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:13,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:13,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:13,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653853304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:13,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653853304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:13,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,309 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,310 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:13,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:13,310 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:13,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653853304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:13,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:13,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:13,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653853504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:13,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:13,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1732653853514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:13,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653853514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:13,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653853514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:13,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653853514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:13,574 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:13,577 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126834851ebf8be4a11b77bdf646df7fc6c_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126834851ebf8be4a11b77bdf646df7fc6c_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:13,578 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/ee24d077689e475dbfc287b5dbf7f5de, store: [table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:13,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/ee24d077689e475dbfc287b5dbf7f5de is 175, key is test_row_0/A:col10/1732653792015/Put/seqid=0 2024-11-26T20:43:13,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742355_1531 (size=39549) 2024-11-26T20:43:13,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T20:43:13,765 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:43:13,765 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:43:13,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-26T20:43:13,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T20:43:13,767 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:43:13,768 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:43:13,768 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:43:13,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:13,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653853814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:13,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:13,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1732653853824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:13,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:13,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:13,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653853824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:13,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:13,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653853824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:13,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:13,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653853824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:13,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T20:43:13,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T20:43:13,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:13,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:13,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:13,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:13,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:13,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:13,983 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/ee24d077689e475dbfc287b5dbf7f5de 2024-11-26T20:43:13,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/00d78b24c0d242ebaac2fe9ec66b4595 is 50, key is test_row_0/B:col10/1732653792015/Put/seqid=0 2024-11-26T20:43:13,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742356_1532 (size=12001) 2024-11-26T20:43:14,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T20:43:14,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:14,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:14,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:14,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:14,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T20:43:14,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T20:43:14,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:14,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:14,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:14,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:14,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:14,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653854324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:14,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:14,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:14,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:14,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:14,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1732653854333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:14,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:14,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653854333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:14,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:14,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653854333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:14,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:14,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653854334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:14,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T20:43:14,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:14,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:14,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:14,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:14,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:14,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/00d78b24c0d242ebaac2fe9ec66b4595 2024-11-26T20:43:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T20:43:14,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/dd5dc25dec4e48929e0cee871cc5ab38 is 50, key is test_row_0/C:col10/1732653792015/Put/seqid=0 2024-11-26T20:43:14,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742357_1533 (size=12001) 2024-11-26T20:43:14,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T20:43:14,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:14,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:14,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:14,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:14,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:14,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:14,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T20:43:14,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:14,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:14,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:14,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:14,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:14,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/dd5dc25dec4e48929e0cee871cc5ab38 2024-11-26T20:43:14,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/ee24d077689e475dbfc287b5dbf7f5de as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/ee24d077689e475dbfc287b5dbf7f5de 2024-11-26T20:43:14,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/ee24d077689e475dbfc287b5dbf7f5de, entries=200, sequenceid=64, filesize=38.6 K 2024-11-26T20:43:14,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/00d78b24c0d242ebaac2fe9ec66b4595 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/00d78b24c0d242ebaac2fe9ec66b4595 2024-11-26T20:43:14,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/00d78b24c0d242ebaac2fe9ec66b4595, entries=150, sequenceid=64, filesize=11.7 K 2024-11-26T20:43:14,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/dd5dc25dec4e48929e0cee871cc5ab38 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/dd5dc25dec4e48929e0cee871cc5ab38 2024-11-26T20:43:14,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/dd5dc25dec4e48929e0cee871cc5ab38, entries=150, sequenceid=64, filesize=11.7 K 2024-11-26T20:43:14,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=114.05 KB/116790 for 0c644311c9706b1426d8809c603e0fe8 in 1662ms, sequenceid=64, compaction requested=true 2024-11-26T20:43:14,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:14,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0c644311c9706b1426d8809c603e0fe8:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:43:14,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:14,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0c644311c9706b1426d8809c603e0fe8:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:43:14,819 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:43:14,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:14,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0c644311c9706b1426d8809c603e0fe8:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:43:14,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:43:14,819 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:43:14,820 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:43:14,820 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 0c644311c9706b1426d8809c603e0fe8/B is initiating minor compaction (all files) 2024-11-26T20:43:14,820 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0c644311c9706b1426d8809c603e0fe8/B in TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:14,820 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/bc1d0778308d49659bfc371e4a956c5e, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/ab6460139e8344dd8d0e43f0ef4f35b8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/00d78b24c0d242ebaac2fe9ec66b4595] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp, totalSize=35.2 K 2024-11-26T20:43:14,820 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:43:14,821 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 0c644311c9706b1426d8809c603e0fe8/A is initiating minor compaction (all files) 2024-11-26T20:43:14,821 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0c644311c9706b1426d8809c603e0fe8/A in TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:14,821 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/17aac43a6e224bd2b960c6b48edf9ff9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/bee9024d6d014948b6b583859ecca752, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/ee24d077689e475dbfc287b5dbf7f5de] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp, totalSize=99.1 K 2024-11-26T20:43:14,821 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:14,821 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. files: [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/17aac43a6e224bd2b960c6b48edf9ff9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/bee9024d6d014948b6b583859ecca752, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/ee24d077689e475dbfc287b5dbf7f5de] 2024-11-26T20:43:14,821 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting bc1d0778308d49659bfc371e4a956c5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732653791622 2024-11-26T20:43:14,821 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17aac43a6e224bd2b960c6b48edf9ff9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732653791622 2024-11-26T20:43:14,822 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting ab6460139e8344dd8d0e43f0ef4f35b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732653791677 2024-11-26T20:43:14,822 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting bee9024d6d014948b6b583859ecca752, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732653791677 2024-11-26T20:43:14,822 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 00d78b24c0d242ebaac2fe9ec66b4595, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732653792015 2024-11-26T20:43:14,822 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee24d077689e475dbfc287b5dbf7f5de, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732653792015 2024-11-26T20:43:14,827 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:14,828 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0c644311c9706b1426d8809c603e0fe8#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:43:14,829 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411265c2ef552c2f348448d8e5b995509de88_0c644311c9706b1426d8809c603e0fe8 store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:14,829 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/51328fe3d2d74e6583aa9b4f9f808eea is 50, key is test_row_0/B:col10/1732653792015/Put/seqid=0 2024-11-26T20:43:14,831 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411265c2ef552c2f348448d8e5b995509de88_0c644311c9706b1426d8809c603e0fe8, store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:14,831 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265c2ef552c2f348448d8e5b995509de88_0c644311c9706b1426d8809c603e0fe8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:14,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T20:43:14,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:14,837 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 0c644311c9706b1426d8809c603e0fe8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T20:43:14,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=A 2024-11-26T20:43:14,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:14,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=B 2024-11-26T20:43:14,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:14,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=C 2024-11-26T20:43:14,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:14,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742358_1534 (size=12104) 2024-11-26T20:43:14,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e4d479bf0e1f4e81948412fae76e3d8f_0c644311c9706b1426d8809c603e0fe8 is 50, key is test_row_0/A:col10/1732653793186/Put/seqid=0 2024-11-26T20:43:14,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742359_1535 (size=4469) 2024-11-26T20:43:14,856 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/51328fe3d2d74e6583aa9b4f9f808eea as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/51328fe3d2d74e6583aa9b4f9f808eea 2024-11-26T20:43:14,860 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0c644311c9706b1426d8809c603e0fe8/B of 0c644311c9706b1426d8809c603e0fe8 into 51328fe3d2d74e6583aa9b4f9f808eea(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:43:14,860 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:14,860 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., storeName=0c644311c9706b1426d8809c603e0fe8/B, priority=13, startTime=1732653794819; duration=0sec 2024-11-26T20:43:14,860 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:43:14,860 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0c644311c9706b1426d8809c603e0fe8:B 2024-11-26T20:43:14,860 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:43:14,861 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:43:14,861 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 0c644311c9706b1426d8809c603e0fe8/C is initiating minor compaction (all files) 2024-11-26T20:43:14,861 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0c644311c9706b1426d8809c603e0fe8/C in TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:14,861 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/ae1754899b0f41169026ae72cda10d84, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/fc0130d442304d87b84f419db3efa305, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/dd5dc25dec4e48929e0cee871cc5ab38] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp, totalSize=35.2 K 2024-11-26T20:43:14,862 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting ae1754899b0f41169026ae72cda10d84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732653791622 2024-11-26T20:43:14,862 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting fc0130d442304d87b84f419db3efa305, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732653791677 2024-11-26T20:43:14,862 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting dd5dc25dec4e48929e0cee871cc5ab38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732653792015 2024-11-26T20:43:14,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742360_1536 (size=12154) 2024-11-26T20:43:14,868 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0c644311c9706b1426d8809c603e0fe8#C#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:43:14,869 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/a396f3de1635436e99f7a51e06fe7b4f is 50, key is test_row_0/C:col10/1732653792015/Put/seqid=0 2024-11-26T20:43:14,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742361_1537 (size=12104) 2024-11-26T20:43:14,881 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/a396f3de1635436e99f7a51e06fe7b4f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/a396f3de1635436e99f7a51e06fe7b4f 2024-11-26T20:43:14,885 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0c644311c9706b1426d8809c603e0fe8/C of 0c644311c9706b1426d8809c603e0fe8 into a396f3de1635436e99f7a51e06fe7b4f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:43:14,885 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:14,885 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., storeName=0c644311c9706b1426d8809c603e0fe8/C, priority=13, startTime=1732653794819; duration=0sec 2024-11-26T20:43:14,885 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:14,885 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0c644311c9706b1426d8809c603e0fe8:C 2024-11-26T20:43:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T20:43:15,254 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0c644311c9706b1426d8809c603e0fe8#A#compaction#460 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:43:15,255 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/c1222c836f3a4077a59f6617d41b8ad8 is 175, key is test_row_0/A:col10/1732653792015/Put/seqid=0 2024-11-26T20:43:15,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:15,269 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e4d479bf0e1f4e81948412fae76e3d8f_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e4d479bf0e1f4e81948412fae76e3d8f_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:15,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/b09dab8627d64a8d82d3144f208ff361, store: [table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:15,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/b09dab8627d64a8d82d3144f208ff361 is 175, key is test_row_0/A:col10/1732653793186/Put/seqid=0 2024-11-26T20:43:15,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742363_1539 (size=30955) 2024-11-26T20:43:15,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742362_1538 (size=31058) 2024-11-26T20:43:15,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:15,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:15,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653855349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:15,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:15,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653855352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653855353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:15,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1732653855354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:15,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653855355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:15,358 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_2' in 0c644311c9706b1426d8809c603e0fe8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:43:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:15,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653855464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:15,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:15,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653855464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:15,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:15,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:15,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653855464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:15,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:15,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:15,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653855464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:15,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:15,676 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/b09dab8627d64a8d82d3144f208ff361 2024-11-26T20:43:15,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653855674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:15,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:15,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653855674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653855674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653855674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:15,682 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/c1222c836f3a4077a59f6617d41b8ad8 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/c1222c836f3a4077a59f6617d41b8ad8 2024-11-26T20:43:15,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/3048a20b0e674733b60ea4aedf7c2689 is 50, key is test_row_0/B:col10/1732653793186/Put/seqid=0 2024-11-26T20:43:15,686 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0c644311c9706b1426d8809c603e0fe8/A of 0c644311c9706b1426d8809c603e0fe8 into c1222c836f3a4077a59f6617d41b8ad8(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:43:15,686 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:15,686 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., storeName=0c644311c9706b1426d8809c603e0fe8/A, priority=13, startTime=1732653794819; duration=0sec 2024-11-26T20:43:15,686 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:15,686 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0c644311c9706b1426d8809c603e0fe8:A 2024-11-26T20:43:15,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742364_1540 (size=12001) 2024-11-26T20:43:15,689 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/3048a20b0e674733b60ea4aedf7c2689 2024-11-26T20:43:15,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/54fdfa821c4c43e28b7449f571748abf is 50, key is test_row_0/C:col10/1732653793186/Put/seqid=0 2024-11-26T20:43:15,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742365_1541 (size=12001) 2024-11-26T20:43:15,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T20:43:15,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653855984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:15,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653855984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:15,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653855984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:15,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653855984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:16,099 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/54fdfa821c4c43e28b7449f571748abf 2024-11-26T20:43:16,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/b09dab8627d64a8d82d3144f208ff361 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/b09dab8627d64a8d82d3144f208ff361 2024-11-26T20:43:16,106 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/b09dab8627d64a8d82d3144f208ff361, entries=150, sequenceid=84, filesize=30.2 K 2024-11-26T20:43:16,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/3048a20b0e674733b60ea4aedf7c2689 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/3048a20b0e674733b60ea4aedf7c2689 2024-11-26T20:43:16,110 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/3048a20b0e674733b60ea4aedf7c2689, entries=150, sequenceid=84, filesize=11.7 K 2024-11-26T20:43:16,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/54fdfa821c4c43e28b7449f571748abf as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/54fdfa821c4c43e28b7449f571748abf 2024-11-26T20:43:16,113 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/54fdfa821c4c43e28b7449f571748abf, entries=150, sequenceid=84, filesize=11.7 K 2024-11-26T20:43:16,114 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 0c644311c9706b1426d8809c603e0fe8 in 1277ms, sequenceid=84, compaction requested=false 2024-11-26T20:43:16,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:16,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:16,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-26T20:43:16,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-26T20:43:16,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-26T20:43:16,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3490 sec 2024-11-26T20:43:16,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.3540 sec 2024-11-26T20:43:16,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:16,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0c644311c9706b1426d8809c603e0fe8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T20:43:16,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=A 2024-11-26T20:43:16,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:16,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=B 2024-11-26T20:43:16,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:16,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=C 2024-11-26T20:43:16,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:16,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ac0fc2ffb61a4bcd9c807198dcb7a620_0c644311c9706b1426d8809c603e0fe8 is 50, key is test_row_0/A:col10/1732653796497/Put/seqid=0 2024-11-26T20:43:16,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742366_1542 (size=12154) 2024-11-26T20:43:16,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653856517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653856518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653856519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:16,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653856520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:16,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:16,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653856624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:16,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653856624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:16,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653856634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653856634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:16,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653856834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:16,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653856834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:16,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:16,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:16,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653856844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:16,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653856844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:16,910 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:16,913 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ac0fc2ffb61a4bcd9c807198dcb7a620_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ac0fc2ffb61a4bcd9c807198dcb7a620_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:16,914 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/60c4b3bdd6a54e459c471bca76995f0d, store: [table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:16,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/60c4b3bdd6a54e459c471bca76995f0d is 175, key is test_row_0/A:col10/1732653796497/Put/seqid=0 2024-11-26T20:43:16,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742367_1543 (size=30955) 2024-11-26T20:43:17,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653857144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:17,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:17,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:17,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653857144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:17,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:17,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:17,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:17,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:17,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653857154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:17,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:17,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653857154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:17,318 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=102, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/60c4b3bdd6a54e459c471bca76995f0d 2024-11-26T20:43:17,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/0dd6916ccfdf4b1fb875966853fe7b93 is 50, key is test_row_0/B:col10/1732653796497/Put/seqid=0 2024-11-26T20:43:17,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742368_1544 (size=12001) 2024-11-26T20:43:17,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1732653857373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:17,375 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_2' in 0c644311c9706b1426d8809c603e0fe8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:43:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:17,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:17,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653857653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:17,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:17,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:17,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:17,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:17,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653857654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:17,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:17,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:17,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:17,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:17,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653857664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:17,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:17,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653857664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:17,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/0dd6916ccfdf4b1fb875966853fe7b93 2024-11-26T20:43:17,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/cdce5f863f7e40e1afa1987d32b22cda is 50, key is test_row_0/C:col10/1732653796497/Put/seqid=0 2024-11-26T20:43:17,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742369_1545 (size=12001) 2024-11-26T20:43:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T20:43:17,924 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:43:17,925 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:43:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-26T20:43:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T20:43:17,926 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:43:17,927 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:43:17,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:43:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T20:43:18,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-26T20:43:18,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:18,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:18,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:18,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:18,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:18,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/cdce5f863f7e40e1afa1987d32b22cda 2024-11-26T20:43:18,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/60c4b3bdd6a54e459c471bca76995f0d as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/60c4b3bdd6a54e459c471bca76995f0d 2024-11-26T20:43:18,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/60c4b3bdd6a54e459c471bca76995f0d, entries=150, sequenceid=102, filesize=30.2 K 2024-11-26T20:43:18,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/0dd6916ccfdf4b1fb875966853fe7b93 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/0dd6916ccfdf4b1fb875966853fe7b93 2024-11-26T20:43:18,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/0dd6916ccfdf4b1fb875966853fe7b93, entries=150, sequenceid=102, filesize=11.7 K 2024-11-26T20:43:18,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/cdce5f863f7e40e1afa1987d32b22cda as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/cdce5f863f7e40e1afa1987d32b22cda 2024-11-26T20:43:18,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/cdce5f863f7e40e1afa1987d32b22cda, entries=150, sequenceid=102, filesize=11.7 K 2024-11-26T20:43:18,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 0c644311c9706b1426d8809c603e0fe8 in 1654ms, sequenceid=102, compaction requested=true 2024-11-26T20:43:18,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:18,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0c644311c9706b1426d8809c603e0fe8:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:43:18,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:18,153 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:43:18,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0c644311c9706b1426d8809c603e0fe8:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:43:18,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:18,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0c644311c9706b1426d8809c603e0fe8:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:43:18,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:43:18,153 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:43:18,156 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:43:18,156 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:43:18,156 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 0c644311c9706b1426d8809c603e0fe8/B is initiating minor compaction (all files) 2024-11-26T20:43:18,156 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0c644311c9706b1426d8809c603e0fe8/B in TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:18,156 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 0c644311c9706b1426d8809c603e0fe8/A is initiating minor compaction (all files) 2024-11-26T20:43:18,156 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/51328fe3d2d74e6583aa9b4f9f808eea, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/3048a20b0e674733b60ea4aedf7c2689, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/0dd6916ccfdf4b1fb875966853fe7b93] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp, totalSize=35.3 K 2024-11-26T20:43:18,156 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0c644311c9706b1426d8809c603e0fe8/A in TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:18,157 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/c1222c836f3a4077a59f6617d41b8ad8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/b09dab8627d64a8d82d3144f208ff361, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/60c4b3bdd6a54e459c471bca76995f0d] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp, totalSize=90.8 K 2024-11-26T20:43:18,157 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:18,157 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. files: [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/c1222c836f3a4077a59f6617d41b8ad8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/b09dab8627d64a8d82d3144f208ff361, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/60c4b3bdd6a54e459c471bca76995f0d] 2024-11-26T20:43:18,157 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 51328fe3d2d74e6583aa9b4f9f808eea, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732653792015 2024-11-26T20:43:18,157 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1222c836f3a4077a59f6617d41b8ad8, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732653792015 2024-11-26T20:43:18,157 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting b09dab8627d64a8d82d3144f208ff361, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732653793166 2024-11-26T20:43:18,157 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 3048a20b0e674733b60ea4aedf7c2689, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732653793166 2024-11-26T20:43:18,158 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60c4b3bdd6a54e459c471bca76995f0d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732653796496 2024-11-26T20:43:18,158 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dd6916ccfdf4b1fb875966853fe7b93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732653796496 2024-11-26T20:43:18,162 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:18,164 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0c644311c9706b1426d8809c603e0fe8#B#compaction#468 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:43:18,164 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/2ca4f86aacc24d63ba2bd0febfa3c47f is 50, key is test_row_0/B:col10/1732653796497/Put/seqid=0 2024-11-26T20:43:18,178 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126a9c8d667d156400f8a9c6bf2dfd1279f_0c644311c9706b1426d8809c603e0fe8 store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:18,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742370_1546 (size=12207) 2024-11-26T20:43:18,179 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126a9c8d667d156400f8a9c6bf2dfd1279f_0c644311c9706b1426d8809c603e0fe8, store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:18,180 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a9c8d667d156400f8a9c6bf2dfd1279f_0c644311c9706b1426d8809c603e0fe8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:18,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742371_1547 (size=4469) 2024-11-26T20:43:18,186 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0c644311c9706b1426d8809c603e0fe8#A#compaction#469 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:43:18,187 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/2ca4f86aacc24d63ba2bd0febfa3c47f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/2ca4f86aacc24d63ba2bd0febfa3c47f 2024-11-26T20:43:18,187 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/30c0567e74c9405199f5c695c8f73052 is 175, key is test_row_0/A:col10/1732653796497/Put/seqid=0 2024-11-26T20:43:18,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742372_1548 (size=31161) 2024-11-26T20:43:18,193 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0c644311c9706b1426d8809c603e0fe8/B of 0c644311c9706b1426d8809c603e0fe8 into 2ca4f86aacc24d63ba2bd0febfa3c47f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:43:18,193 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:18,193 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., storeName=0c644311c9706b1426d8809c603e0fe8/B, priority=13, startTime=1732653798153; duration=0sec 2024-11-26T20:43:18,193 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:43:18,193 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0c644311c9706b1426d8809c603e0fe8:B 2024-11-26T20:43:18,193 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:43:18,194 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:43:18,194 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 0c644311c9706b1426d8809c603e0fe8/C is initiating minor compaction (all files) 2024-11-26T20:43:18,194 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0c644311c9706b1426d8809c603e0fe8/C in TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:18,194 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/a396f3de1635436e99f7a51e06fe7b4f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/54fdfa821c4c43e28b7449f571748abf, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/cdce5f863f7e40e1afa1987d32b22cda] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp, totalSize=35.3 K 2024-11-26T20:43:18,195 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/30c0567e74c9405199f5c695c8f73052 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/30c0567e74c9405199f5c695c8f73052 2024-11-26T20:43:18,195 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting a396f3de1635436e99f7a51e06fe7b4f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732653792015 2024-11-26T20:43:18,195 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 54fdfa821c4c43e28b7449f571748abf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732653793166 2024-11-26T20:43:18,195 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting cdce5f863f7e40e1afa1987d32b22cda, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732653796496 2024-11-26T20:43:18,199 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0c644311c9706b1426d8809c603e0fe8/A of 0c644311c9706b1426d8809c603e0fe8 into 30c0567e74c9405199f5c695c8f73052(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:43:18,199 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:18,199 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., storeName=0c644311c9706b1426d8809c603e0fe8/A, priority=13, startTime=1732653798153; duration=0sec 2024-11-26T20:43:18,199 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:18,199 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0c644311c9706b1426d8809c603e0fe8:A 2024-11-26T20:43:18,203 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0c644311c9706b1426d8809c603e0fe8#C#compaction#470 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:43:18,203 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/10de564917394abfb679a9cba2349dcc is 50, key is test_row_0/C:col10/1732653796497/Put/seqid=0 2024-11-26T20:43:18,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742373_1549 (size=12207) 2024-11-26T20:43:18,210 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/10de564917394abfb679a9cba2349dcc as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/10de564917394abfb679a9cba2349dcc 2024-11-26T20:43:18,213 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0c644311c9706b1426d8809c603e0fe8/C of 0c644311c9706b1426d8809c603e0fe8 into 10de564917394abfb679a9cba2349dcc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:43:18,213 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:18,213 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., storeName=0c644311c9706b1426d8809c603e0fe8/C, priority=13, startTime=1732653798153; duration=0sec 2024-11-26T20:43:18,214 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:18,214 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0c644311c9706b1426d8809c603e0fe8:C 2024-11-26T20:43:18,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-26T20:43:18,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:18,233 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 0c644311c9706b1426d8809c603e0fe8 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T20:43:18,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=A 2024-11-26T20:43:18,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:18,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=B 2024-11-26T20:43:18,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:18,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=C 2024-11-26T20:43:18,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:18,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112652fb6dd90ccd4332a060868fa534ea02_0c644311c9706b1426d8809c603e0fe8 is 50, key is test_row_0/A:col10/1732653796515/Put/seqid=0 2024-11-26T20:43:18,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742374_1550 (size=12154) 2024-11-26T20:43:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T20:43:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T20:43:18,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:18,647 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112652fb6dd90ccd4332a060868fa534ea02_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112652fb6dd90ccd4332a060868fa534ea02_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:18,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/09e85645a4814c8ea16c4bc194533159, store: [table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:18,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/09e85645a4814c8ea16c4bc194533159 is 175, key is test_row_0/A:col10/1732653796515/Put/seqid=0 2024-11-26T20:43:18,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742375_1551 (size=30955) 2024-11-26T20:43:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:18,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:18,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653858684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:18,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653858684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:18,688 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 0c644311c9706b1426d8809c603e0fe8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:43:18,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:18,688 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 0c644311c9706b1426d8809c603e0fe8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:43:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653858686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:18,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:18,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:18,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:18,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:18,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:18,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653858687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:18,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653858794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:18,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653858794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:19,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653859004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:19,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653859004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:19,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:19,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:19,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:19,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:19,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:19,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:19,053 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/09e85645a4814c8ea16c4bc194533159 2024-11-26T20:43:19,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/9da92ffc94484f20be5b1aaf61edede3 is 50, key is test_row_0/B:col10/1732653796515/Put/seqid=0 2024-11-26T20:43:19,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742376_1552 (size=12001) 2024-11-26T20:43:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T20:43:19,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:19,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:19,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653859314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:19,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653859313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:19,462 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/9da92ffc94484f20be5b1aaf61edede3 2024-11-26T20:43:19,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/6ea1109aec7448d19683fb9e11c9d181 is 50, key is test_row_0/C:col10/1732653796515/Put/seqid=0 2024-11-26T20:43:19,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742377_1553 (size=12001) 2024-11-26T20:43:19,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:19,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653859823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:19,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:19,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653859823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:19,873 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/6ea1109aec7448d19683fb9e11c9d181 2024-11-26T20:43:19,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/09e85645a4814c8ea16c4bc194533159 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/09e85645a4814c8ea16c4bc194533159 2024-11-26T20:43:19,880 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/09e85645a4814c8ea16c4bc194533159, entries=150, sequenceid=122, filesize=30.2 K 2024-11-26T20:43:19,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/9da92ffc94484f20be5b1aaf61edede3 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/9da92ffc94484f20be5b1aaf61edede3 2024-11-26T20:43:19,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,884 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/9da92ffc94484f20be5b1aaf61edede3, entries=150, sequenceid=122, filesize=11.7 K 2024-11-26T20:43:19,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/6ea1109aec7448d19683fb9e11c9d181 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/6ea1109aec7448d19683fb9e11c9d181 2024-11-26T20:43:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,887 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/6ea1109aec7448d19683fb9e11c9d181, entries=150, sequenceid=122, filesize=11.7 K 2024-11-26T20:43:19,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,888 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 0c644311c9706b1426d8809c603e0fe8 in 1655ms, sequenceid=122, compaction requested=false 2024-11-26T20:43:19,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:19,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:19,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-26T20:43:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-26T20:43:19,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-26T20:43:19,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9630 sec 2024-11-26T20:43:19,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.9670 sec 2024-11-26T20:43:19,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:19,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T20:43:20,074 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:43:20,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,074 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:43:20,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-26T20:43:20,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,077 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:43:20,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,078 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:43:20,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:43:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T20:43:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T20:43:20,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-26T20:43:20,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,231 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 0c644311c9706b1426d8809c603e0fe8 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T20:43:20,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=A 2024-11-26T20:43:20,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:20,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=B 2024-11-26T20:43:20,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:20,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=C 2024-11-26T20:43:20,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:20,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d6bb381e600f4386921d10bb7766de39_0c644311c9706b1426d8809c603e0fe8 is 50, key is test_row_0/A:col10/1732653798686/Put/seqid=0 2024-11-26T20:43:20,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742378_1554 (size=9814) 2024-11-26T20:43:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,261 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d6bb381e600f4386921d10bb7766de39_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d6bb381e600f4386921d10bb7766de39_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:20,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/25a45f9193d8481aa3ad2059b3c1cf6f, store: [table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/25a45f9193d8481aa3ad2059b3c1cf6f is 175, key is test_row_0/A:col10/1732653798686/Put/seqid=0 2024-11-26T20:43:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742379_1555 (size=22461) 2024-11-26T20:43:20,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,281 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/25a45f9193d8481aa3ad2059b3c1cf6f 2024-11-26T20:43:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/f1f0abb83b4c4935b313f1db16201eb5 is 50, key is test_row_0/B:col10/1732653798686/Put/seqid=0 2024-11-26T20:43:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742380_1556 (size=9757) 2024-11-26T20:43:20,299 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/f1f0abb83b4c4935b313f1db16201eb5 2024-11-26T20:43:20,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/ce260cc9f0fc44fba40b86e3d651e4bd is 50, key is test_row_0/C:col10/1732653798686/Put/seqid=0 2024-11-26T20:43:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742381_1557 (size=9757) 2024-11-26T20:43:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,324 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/ce260cc9f0fc44fba40b86e3d651e4bd 2024-11-26T20:43:20,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/25a45f9193d8481aa3ad2059b3c1cf6f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/25a45f9193d8481aa3ad2059b3c1cf6f 2024-11-26T20:43:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,336 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/25a45f9193d8481aa3ad2059b3c1cf6f, entries=100, sequenceid=137, filesize=21.9 K 2024-11-26T20:43:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/f1f0abb83b4c4935b313f1db16201eb5 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/f1f0abb83b4c4935b313f1db16201eb5 2024-11-26T20:43:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,341 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/f1f0abb83b4c4935b313f1db16201eb5, entries=100, sequenceid=137, filesize=9.5 K 2024-11-26T20:43:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/ce260cc9f0fc44fba40b86e3d651e4bd as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/ce260cc9f0fc44fba40b86e3d651e4bd 2024-11-26T20:43:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,346 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/ce260cc9f0fc44fba40b86e3d651e4bd, entries=100, sequenceid=137, filesize=9.5 K 2024-11-26T20:43:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,347 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 0c644311c9706b1426d8809c603e0fe8 in 116ms, sequenceid=137, compaction requested=true 2024-11-26T20:43:20,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:20,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-26T20:43:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-26T20:43:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-26T20:43:20,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 271 msec 2024-11-26T20:43:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,353 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 278 msec 2024-11-26T20:43:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T20:43:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,394 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:43:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,395 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:43:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-26T20:43:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T20:43:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,397 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:43:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,398 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:43:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,398 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:43:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T20:43:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-26T20:43:20,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:20,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:20,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:20,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-26T20:43:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-26T20:43:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,555 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-26T20:43:20,555 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-26T20:43:20,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 161 msec 2024-11-26T20:43:20,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T20:43:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,714 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:43:20,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,715 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:43:20,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-26T20:43:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T20:43:20,717 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:43:20,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,718 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:43:20,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:43:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:20,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0c644311c9706b1426d8809c603e0fe8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T20:43:20,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=A 2024-11-26T20:43:20,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:20,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=B 2024-11-26T20:43:20,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:20,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=C 2024-11-26T20:43:20,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:20,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411262c7d43b9e34140e99ca0c1fa4c3bd3d6_0c644311c9706b1426d8809c603e0fe8 is 50, key is test_row_0/A:col10/1732653800753/Put/seqid=0 2024-11-26T20:43:20,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742382_1558 (size=17284) 2024-11-26T20:43:20,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T20:43:20,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653800807, max timestamp=1732653800822Num uniques 859; 2024-11-26T20:43:20,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:43:20,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653800807, max timestamp=1732653800822Num uniques 859; 2024-11-26T20:43:20,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:43:20,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653800807, max timestamp=1732653800822Num uniques 859; 2024-11-26T20:43:20,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:43:20,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:20,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653860852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:20,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:20,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:20,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:20,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:20,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653860852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:20,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:20,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653860853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:20,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:20,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653860854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:20,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T20:43:20,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:20,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:20,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:20,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:20,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:20,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653860964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:20,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653860964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:20,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:20,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653860965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:20,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653860965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:21,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T20:43:21,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:21,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:21,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:21,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:21,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T20:43:21,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:21,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653861173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:21,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:21,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:21,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653861173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:21,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:21,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:21,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:21,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:21,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653861174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:21,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:21,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:21,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:21,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:21,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:21,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:21,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653861174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:21,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:21,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:21,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:21,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T20:43:21,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:21,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:21,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:21,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:21,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:21,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:21,185 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:21,188 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411262c7d43b9e34140e99ca0c1fa4c3bd3d6_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262c7d43b9e34140e99ca0c1fa4c3bd3d6_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:21,189 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/6ab791b8bdf548b1be619762f3366e71, store: [table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:21,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/6ab791b8bdf548b1be619762f3366e71 is 175, key is test_row_0/A:col10/1732653800753/Put/seqid=0 2024-11-26T20:43:21,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742383_1559 (size=48385) 2024-11-26T20:43:21,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T20:43:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:21,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:21,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T20:43:21,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:21,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1732653861383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:21,387 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_2' in 0c644311c9706b1426d8809c603e0fe8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:43:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:21,414 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-26T20:43:21,414 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-26T20:43:21,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T20:43:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:21,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:21,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653861483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:21,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653861484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:21,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653861484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:21,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653861484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:21,593 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=148, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/6ab791b8bdf548b1be619762f3366e71 2024-11-26T20:43:21,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/4033bfbd06db484f8c6803b1bcf1bca4 is 50, key is test_row_0/B:col10/1732653800753/Put/seqid=0 2024-11-26T20:43:21,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742384_1560 (size=9757) 2024-11-26T20:43:21,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T20:43:21,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:21,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:21,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:21,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:21,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:21,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T20:43:21,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:21,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:21,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:21,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:21,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T20:43:21,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T20:43:21,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:21,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:21,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:21,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:21,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:21,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653861994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:21,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:21,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653861994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:21,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:21,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:22,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:22,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:22,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:22,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/4033bfbd06db484f8c6803b1bcf1bca4 2024-11-26T20:43:22,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653862004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:22,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653862004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:22,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/e0f48e35ee584629b891f59e0870238f is 50, key is test_row_0/C:col10/1732653800753/Put/seqid=0 2024-11-26T20:43:22,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742385_1561 (size=9757) 2024-11-26T20:43:22,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T20:43:22,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:22,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:22,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:22,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:22,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:22,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T20:43:22,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:22,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:22,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:22,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:22,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:22,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T20:43:22,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:22,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:22,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:22,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:22,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:22,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/e0f48e35ee584629b891f59e0870238f 2024-11-26T20:43:22,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/6ab791b8bdf548b1be619762f3366e71 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/6ab791b8bdf548b1be619762f3366e71 2024-11-26T20:43:22,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/6ab791b8bdf548b1be619762f3366e71, entries=250, sequenceid=148, filesize=47.3 K 2024-11-26T20:43:22,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/4033bfbd06db484f8c6803b1bcf1bca4 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/4033bfbd06db484f8c6803b1bcf1bca4 2024-11-26T20:43:22,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/4033bfbd06db484f8c6803b1bcf1bca4, entries=100, sequenceid=148, filesize=9.5 K 2024-11-26T20:43:22,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/e0f48e35ee584629b891f59e0870238f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/e0f48e35ee584629b891f59e0870238f 2024-11-26T20:43:22,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/e0f48e35ee584629b891f59e0870238f, entries=100, sequenceid=148, filesize=9.5 K 2024-11-26T20:43:22,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0c644311c9706b1426d8809c603e0fe8 in 1658ms, sequenceid=148, compaction requested=true 2024-11-26T20:43:22,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:22,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0c644311c9706b1426d8809c603e0fe8:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:43:22,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:22,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0c644311c9706b1426d8809c603e0fe8:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:43:22,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:22,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0c644311c9706b1426d8809c603e0fe8:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:43:22,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:43:22,430 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:43:22,430 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:43:22,431 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132962 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:43:22,431 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43722 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:43:22,431 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 0c644311c9706b1426d8809c603e0fe8/B is initiating minor compaction (all files) 2024-11-26T20:43:22,431 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 0c644311c9706b1426d8809c603e0fe8/A is initiating minor compaction (all files) 2024-11-26T20:43:22,431 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0c644311c9706b1426d8809c603e0fe8/A in TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:22,431 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0c644311c9706b1426d8809c603e0fe8/B in TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:22,431 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/30c0567e74c9405199f5c695c8f73052, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/09e85645a4814c8ea16c4bc194533159, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/25a45f9193d8481aa3ad2059b3c1cf6f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/6ab791b8bdf548b1be619762f3366e71] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp, totalSize=129.8 K 2024-11-26T20:43:22,431 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/2ca4f86aacc24d63ba2bd0febfa3c47f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/9da92ffc94484f20be5b1aaf61edede3, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/f1f0abb83b4c4935b313f1db16201eb5, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/4033bfbd06db484f8c6803b1bcf1bca4] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp, totalSize=42.7 K 2024-11-26T20:43:22,431 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:22,431 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. files: [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/30c0567e74c9405199f5c695c8f73052, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/09e85645a4814c8ea16c4bc194533159, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/25a45f9193d8481aa3ad2059b3c1cf6f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/6ab791b8bdf548b1be619762f3366e71] 2024-11-26T20:43:22,432 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30c0567e74c9405199f5c695c8f73052, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732653796496 2024-11-26T20:43:22,432 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ca4f86aacc24d63ba2bd0febfa3c47f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732653796496 2024-11-26T20:43:22,433 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09e85645a4814c8ea16c4bc194533159, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732653796515 2024-11-26T20:43:22,433 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 9da92ffc94484f20be5b1aaf61edede3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732653796515 2024-11-26T20:43:22,433 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting f1f0abb83b4c4935b313f1db16201eb5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732653798673 2024-11-26T20:43:22,433 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25a45f9193d8481aa3ad2059b3c1cf6f, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732653798673 2024-11-26T20:43:22,433 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 4033bfbd06db484f8c6803b1bcf1bca4, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732653800753 2024-11-26T20:43:22,433 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ab791b8bdf548b1be619762f3366e71, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732653800743 2024-11-26T20:43:22,439 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:22,440 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0c644311c9706b1426d8809c603e0fe8#B#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:43:22,440 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/a5e32d22ef3047b2a3729e8949e5a93a is 50, key is test_row_0/B:col10/1732653800753/Put/seqid=0 2024-11-26T20:43:22,441 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411262cb1f1f218344efbbe1c1d0ce2cf44ca_0c644311c9706b1426d8809c603e0fe8 store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:22,444 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411262cb1f1f218344efbbe1c1d0ce2cf44ca_0c644311c9706b1426d8809c603e0fe8, store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:22,444 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411262cb1f1f218344efbbe1c1d0ce2cf44ca_0c644311c9706b1426d8809c603e0fe8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:22,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742386_1562 (size=12493) 2024-11-26T20:43:22,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742387_1563 (size=4469) 2024-11-26T20:43:22,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T20:43:22,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:22,553 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 0c644311c9706b1426d8809c603e0fe8 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-26T20:43:22,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=A 2024-11-26T20:43:22,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:43:22,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=B 2024-11-26T20:43:22,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:43:22,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=C 2024-11-26T20:43:22,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:43:22,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d76f2f278ee841efa96707a20f630ee2_0c644311c9706b1426d8809c603e0fe8 is 50, key is test_row_0/A:col10/1732653800851/Put/seqid=0 2024-11-26T20:43:22,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742388_1564 (size=12304) 2024-11-26T20:43:22,849 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0c644311c9706b1426d8809c603e0fe8#A#compaction#481 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:43:22,850 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/c65dad2614c942209f68fffbd637a63a is 175, key is test_row_0/A:col10/1732653800753/Put/seqid=0 2024-11-26T20:43:22,851 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/a5e32d22ef3047b2a3729e8949e5a93a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/a5e32d22ef3047b2a3729e8949e5a93a 2024-11-26T20:43:22,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742389_1565 (size=31554) 2024-11-26T20:43:22,860 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0c644311c9706b1426d8809c603e0fe8/B of 0c644311c9706b1426d8809c603e0fe8 into a5e32d22ef3047b2a3729e8949e5a93a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:43:22,860 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:22,860 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., storeName=0c644311c9706b1426d8809c603e0fe8/B, priority=12, startTime=1732653802430; duration=0sec 2024-11-26T20:43:22,860 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:43:22,860 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0c644311c9706b1426d8809c603e0fe8:B 2024-11-26T20:43:22,860 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:43:22,861 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43722 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:43:22,861 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 0c644311c9706b1426d8809c603e0fe8/C is initiating minor compaction (all files) 2024-11-26T20:43:22,861 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0c644311c9706b1426d8809c603e0fe8/C in TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:22,861 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/10de564917394abfb679a9cba2349dcc, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/6ea1109aec7448d19683fb9e11c9d181, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/ce260cc9f0fc44fba40b86e3d651e4bd, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/e0f48e35ee584629b891f59e0870238f] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp, totalSize=42.7 K 2024-11-26T20:43:22,861 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 10de564917394abfb679a9cba2349dcc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732653796496 2024-11-26T20:43:22,862 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ea1109aec7448d19683fb9e11c9d181, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732653796515 2024-11-26T20:43:22,862 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting ce260cc9f0fc44fba40b86e3d651e4bd, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732653798673 2024-11-26T20:43:22,862 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting e0f48e35ee584629b891f59e0870238f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732653800753 2024-11-26T20:43:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T20:43:22,869 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/c65dad2614c942209f68fffbd637a63a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/c65dad2614c942209f68fffbd637a63a 2024-11-26T20:43:22,871 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0c644311c9706b1426d8809c603e0fe8#C#compaction#483 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:43:22,871 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/3acba7b2b44547af885a6c8a00d4516a is 50, key is test_row_0/C:col10/1732653800753/Put/seqid=0 2024-11-26T20:43:22,875 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0c644311c9706b1426d8809c603e0fe8/A of 0c644311c9706b1426d8809c603e0fe8 into c65dad2614c942209f68fffbd637a63a(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:43:22,875 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:22,875 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., storeName=0c644311c9706b1426d8809c603e0fe8/A, priority=12, startTime=1732653802430; duration=0sec 2024-11-26T20:43:22,875 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:22,875 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0c644311c9706b1426d8809c603e0fe8:A 2024-11-26T20:43:22,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742390_1566 (size=12493) 2024-11-26T20:43:22,889 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/3acba7b2b44547af885a6c8a00d4516a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/3acba7b2b44547af885a6c8a00d4516a 2024-11-26T20:43:22,893 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0c644311c9706b1426d8809c603e0fe8/C of 0c644311c9706b1426d8809c603e0fe8 into 3acba7b2b44547af885a6c8a00d4516a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:43:22,893 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:22,893 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., storeName=0c644311c9706b1426d8809c603e0fe8/C, priority=12, startTime=1732653802430; duration=0sec 2024-11-26T20:43:22,893 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:22,893 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0c644311c9706b1426d8809c603e0fe8:C 2024-11-26T20:43:22,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:22,968 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d76f2f278ee841efa96707a20f630ee2_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d76f2f278ee841efa96707a20f630ee2_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:22,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/eb0200d2fef847319c36ca453f19b6d1, store: [table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:22,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/eb0200d2fef847319c36ca453f19b6d1 is 175, key is test_row_0/A:col10/1732653800851/Put/seqid=0 2024-11-26T20:43:22,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742391_1567 (size=31105) 2024-11-26T20:43:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:23,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:23,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653863021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:23,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653863024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:23,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653863024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:23,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:23,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653863025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:23,030 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 0c644311c9706b1426d8809c603e0fe8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:43:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:23,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653863134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:23,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653863134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653863134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:23,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653863344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:23,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653863344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:23,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653863344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:23,374 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=185, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/eb0200d2fef847319c36ca453f19b6d1 2024-11-26T20:43:23,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/96e70c9f070943f6a2eb9fa62b81d037 is 50, key is test_row_0/B:col10/1732653800851/Put/seqid=0 2024-11-26T20:43:23,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742392_1568 (size=12151) 2024-11-26T20:43:23,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653863654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:23,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653863655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653863655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:23,788 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/96e70c9f070943f6a2eb9fa62b81d037 2024-11-26T20:43:23,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/f526a745f73549eda4da59d03688fa98 is 50, key is test_row_0/C:col10/1732653800851/Put/seqid=0 2024-11-26T20:43:23,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742393_1569 (size=12151) 2024-11-26T20:43:23,845 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/f526a745f73549eda4da59d03688fa98 2024-11-26T20:43:23,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/eb0200d2fef847319c36ca453f19b6d1 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/eb0200d2fef847319c36ca453f19b6d1 2024-11-26T20:43:23,856 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/eb0200d2fef847319c36ca453f19b6d1, entries=150, sequenceid=185, filesize=30.4 K 2024-11-26T20:43:23,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/96e70c9f070943f6a2eb9fa62b81d037 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/96e70c9f070943f6a2eb9fa62b81d037 2024-11-26T20:43:23,861 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/96e70c9f070943f6a2eb9fa62b81d037, entries=150, sequenceid=185, filesize=11.9 K 2024-11-26T20:43:23,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/f526a745f73549eda4da59d03688fa98 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/f526a745f73549eda4da59d03688fa98 2024-11-26T20:43:23,865 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/f526a745f73549eda4da59d03688fa98, entries=150, sequenceid=185, filesize=11.9 K 2024-11-26T20:43:23,866 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 0c644311c9706b1426d8809c603e0fe8 in 1312ms, sequenceid=185, compaction requested=false 2024-11-26T20:43:23,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:23,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:23,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-26T20:43:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-26T20:43:23,869 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-26T20:43:23,869 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1490 sec 2024-11-26T20:43:23,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 3.1560 sec 2024-11-26T20:43:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:24,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0c644311c9706b1426d8809c603e0fe8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T20:43:24,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=A 2024-11-26T20:43:24,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:24,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=B 2024-11-26T20:43:24,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:24,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=C 2024-11-26T20:43:24,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:24,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e85426934ed7452babd30a8e0c9ab5db_0c644311c9706b1426d8809c603e0fe8 is 50, key is test_row_0/A:col10/1732653803019/Put/seqid=0 2024-11-26T20:43:24,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742394_1570 (size=17284) 2024-11-26T20:43:24,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:24,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653864203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:24,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:24,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653864204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:24,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:24,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:24,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653864204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:24,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:24,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:24,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653864314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:24,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:24,318 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,318 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653864314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:24,318 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:24,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:24,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653864314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:24,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:24,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653864524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:24,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:24,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653864524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:24,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653864524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:24,578 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:24,581 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e85426934ed7452babd30a8e0c9ab5db_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e85426934ed7452babd30a8e0c9ab5db_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:24,582 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/1106645ffa8e4cccb35d376980e93c7a, store: [table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:24,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/1106645ffa8e4cccb35d376980e93c7a is 175, key is test_row_0/A:col10/1732653803019/Put/seqid=0 2024-11-26T20:43:24,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742395_1571 (size=48389) 2024-11-26T20:43:24,593 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/1106645ffa8e4cccb35d376980e93c7a 2024-11-26T20:43:24,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/b2e1122d37864eba97c71a676ac96636 is 50, key is test_row_0/B:col10/1732653803019/Put/seqid=0 2024-11-26T20:43:24,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742396_1572 (size=12151) 2024-11-26T20:43:24,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/b2e1122d37864eba97c71a676ac96636 2024-11-26T20:43:24,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/7bf4b5e011fc49fc80ce4acbf4316500 is 50, key is test_row_0/C:col10/1732653803019/Put/seqid=0 2024-11-26T20:43:24,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742397_1573 (size=12151) 2024-11-26T20:43:24,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/7bf4b5e011fc49fc80ce4acbf4316500 2024-11-26T20:43:24,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/1106645ffa8e4cccb35d376980e93c7a as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/1106645ffa8e4cccb35d376980e93c7a 2024-11-26T20:43:24,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/1106645ffa8e4cccb35d376980e93c7a, entries=250, sequenceid=201, filesize=47.3 K 2024-11-26T20:43:24,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/b2e1122d37864eba97c71a676ac96636 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/b2e1122d37864eba97c71a676ac96636 2024-11-26T20:43:24,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/b2e1122d37864eba97c71a676ac96636, entries=150, sequenceid=201, filesize=11.9 K 2024-11-26T20:43:24,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/7bf4b5e011fc49fc80ce4acbf4316500 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/7bf4b5e011fc49fc80ce4acbf4316500 2024-11-26T20:43:24,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/7bf4b5e011fc49fc80ce4acbf4316500, entries=150, sequenceid=201, filesize=11.9 K 2024-11-26T20:43:24,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 0c644311c9706b1426d8809c603e0fe8 in 470ms, sequenceid=201, compaction requested=true 2024-11-26T20:43:24,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:24,636 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:43:24,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0c644311c9706b1426d8809c603e0fe8:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:43:24,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:24,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0c644311c9706b1426d8809c603e0fe8:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:43:24,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:24,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0c644311c9706b1426d8809c603e0fe8:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:43:24,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:43:24,637 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:43:24,637 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111048 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:43:24,637 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 0c644311c9706b1426d8809c603e0fe8/A is initiating minor compaction (all files) 2024-11-26T20:43:24,637 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0c644311c9706b1426d8809c603e0fe8/A in TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:24,637 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/c65dad2614c942209f68fffbd637a63a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/eb0200d2fef847319c36ca453f19b6d1, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/1106645ffa8e4cccb35d376980e93c7a] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp, totalSize=108.4 K 2024-11-26T20:43:24,637 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:24,637 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. files: [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/c65dad2614c942209f68fffbd637a63a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/eb0200d2fef847319c36ca453f19b6d1, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/1106645ffa8e4cccb35d376980e93c7a] 2024-11-26T20:43:24,638 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:43:24,638 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 0c644311c9706b1426d8809c603e0fe8/B is initiating minor compaction (all files) 2024-11-26T20:43:24,638 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0c644311c9706b1426d8809c603e0fe8/B in TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:24,638 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/a5e32d22ef3047b2a3729e8949e5a93a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/96e70c9f070943f6a2eb9fa62b81d037, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/b2e1122d37864eba97c71a676ac96636] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp, totalSize=35.9 K 2024-11-26T20:43:24,638 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting c65dad2614c942209f68fffbd637a63a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732653798673 2024-11-26T20:43:24,639 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting a5e32d22ef3047b2a3729e8949e5a93a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732653798673 2024-11-26T20:43:24,639 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb0200d2fef847319c36ca453f19b6d1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732653800851 2024-11-26T20:43:24,639 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 96e70c9f070943f6a2eb9fa62b81d037, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732653800851 2024-11-26T20:43:24,639 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1106645ffa8e4cccb35d376980e93c7a, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732653803019 2024-11-26T20:43:24,639 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting b2e1122d37864eba97c71a676ac96636, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732653803019 2024-11-26T20:43:24,644 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:24,648 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0c644311c9706b1426d8809c603e0fe8#B#compaction#490 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:43:24,649 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/9ea2dc884f99456c996e798660427ece is 50, key is test_row_0/B:col10/1732653803019/Put/seqid=0 2024-11-26T20:43:24,650 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112673d72574980d490c9e605d5d90bf41a7_0c644311c9706b1426d8809c603e0fe8 store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:24,652 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112673d72574980d490c9e605d5d90bf41a7_0c644311c9706b1426d8809c603e0fe8, store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:24,652 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112673d72574980d490c9e605d5d90bf41a7_0c644311c9706b1426d8809c603e0fe8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:24,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742399_1575 (size=4469) 2024-11-26T20:43:24,662 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0c644311c9706b1426d8809c603e0fe8#A#compaction#489 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:43:24,663 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/26ae3b70f2ab49c0ae9c04d179278741 is 175, key is test_row_0/A:col10/1732653803019/Put/seqid=0 2024-11-26T20:43:24,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742398_1574 (size=12595) 2024-11-26T20:43:24,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742400_1576 (size=31549) 2024-11-26T20:43:24,681 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/26ae3b70f2ab49c0ae9c04d179278741 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/26ae3b70f2ab49c0ae9c04d179278741 2024-11-26T20:43:24,686 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0c644311c9706b1426d8809c603e0fe8/A of 0c644311c9706b1426d8809c603e0fe8 into 26ae3b70f2ab49c0ae9c04d179278741(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:43:24,686 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:24,686 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., storeName=0c644311c9706b1426d8809c603e0fe8/A, priority=13, startTime=1732653804636; duration=0sec 2024-11-26T20:43:24,686 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:43:24,686 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0c644311c9706b1426d8809c603e0fe8:A 2024-11-26T20:43:24,686 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T20:43:24,688 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T20:43:24,688 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 0c644311c9706b1426d8809c603e0fe8/C is initiating minor compaction (all files) 2024-11-26T20:43:24,688 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0c644311c9706b1426d8809c603e0fe8/C in TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:24,688 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/3acba7b2b44547af885a6c8a00d4516a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/f526a745f73549eda4da59d03688fa98, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/7bf4b5e011fc49fc80ce4acbf4316500] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp, totalSize=35.9 K 2024-11-26T20:43:24,688 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3acba7b2b44547af885a6c8a00d4516a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732653798673 2024-11-26T20:43:24,688 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting f526a745f73549eda4da59d03688fa98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732653800851 2024-11-26T20:43:24,689 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bf4b5e011fc49fc80ce4acbf4316500, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732653803019 2024-11-26T20:43:24,695 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0c644311c9706b1426d8809c603e0fe8#C#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:43:24,696 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/528123e9fc804c82b16ca8c03809cb61 is 50, key is test_row_0/C:col10/1732653803019/Put/seqid=0 2024-11-26T20:43:24,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742401_1577 (size=12595) 2024-11-26T20:43:24,705 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/528123e9fc804c82b16ca8c03809cb61 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/528123e9fc804c82b16ca8c03809cb61 2024-11-26T20:43:24,709 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0c644311c9706b1426d8809c603e0fe8/C of 0c644311c9706b1426d8809c603e0fe8 into 528123e9fc804c82b16ca8c03809cb61(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:43:24,709 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:24,709 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., storeName=0c644311c9706b1426d8809c603e0fe8/C, priority=13, startTime=1732653804636; duration=0sec 2024-11-26T20:43:24,709 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:24,709 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0c644311c9706b1426d8809c603e0fe8:C 2024-11-26T20:43:24,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0c644311c9706b1426d8809c603e0fe8 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-26T20:43:24,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=A 2024-11-26T20:43:24,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:43:24,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=B 2024-11-26T20:43:24,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:43:24,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=C 2024-11-26T20:43:24,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:43:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:24,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112694f5927233234d35afe1bf3e8fce3a10_0c644311c9706b1426d8809c603e0fe8 is 50, key is test_row_0/A:col10/1732653804835/Put/seqid=0 2024-11-26T20:43:24,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742402_1578 (size=14794) 2024-11-26T20:43:24,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653864850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:24,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:24,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653864851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:24,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:24,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653864852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:24,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T20:43:24,874 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:43:24,874 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:43:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-26T20:43:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T20:43:24,876 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:43:24,876 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:43:24,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:43:24,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653864964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:24,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:24,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653864964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653864964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T20:43:25,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T20:43:25,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:25,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:25,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:25,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:25,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:25,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653865044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:25,048 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 0c644311c9706b1426d8809c603e0fe8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:43:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:25,071 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/9ea2dc884f99456c996e798660427ece as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/9ea2dc884f99456c996e798660427ece 2024-11-26T20:43:25,076 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0c644311c9706b1426d8809c603e0fe8/B of 0c644311c9706b1426d8809c603e0fe8 into 9ea2dc884f99456c996e798660427ece(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:43:25,076 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:25,076 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., storeName=0c644311c9706b1426d8809c603e0fe8/B, priority=13, startTime=1732653804636; duration=0sec 2024-11-26T20:43:25,076 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:25,076 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0c644311c9706b1426d8809c603e0fe8:B 2024-11-26T20:43:25,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:25,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653865174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:25,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:25,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653865174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:25,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:25,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653865174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:25,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T20:43:25,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:25,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:25,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:25,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:25,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:25,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T20:43:25,248 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:25,251 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112694f5927233234d35afe1bf3e8fce3a10_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112694f5927233234d35afe1bf3e8fce3a10_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:25,252 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/df84956101094be3b344331dc83f3d12, store: [table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:25,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/df84956101094be3b344331dc83f3d12 is 175, key is test_row_0/A:col10/1732653804835/Put/seqid=0 2024-11-26T20:43:25,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742403_1579 (size=39749) 2024-11-26T20:43:25,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T20:43:25,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:25,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:25,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:25,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:25,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:25,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:25,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T20:43:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653865484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:25,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:25,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:25,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:25,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:25,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653865484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:25,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:25,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653865484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:25,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:25,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:25,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:25,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T20:43:25,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T20:43:25,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:25,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:25,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:25,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:25,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:25,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:25,657 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/df84956101094be3b344331dc83f3d12 2024-11-26T20:43:25,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/6baa337f0db14c7bbdfaeed4d0f4b3f7 is 50, key is test_row_0/B:col10/1732653804835/Put/seqid=0 2024-11-26T20:43:25,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742404_1580 (size=12151) 2024-11-26T20:43:25,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T20:43:25,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:25,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:25,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:25,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:25,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:25,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T20:43:25,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:25,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:25,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:25,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:25,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:25,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653865994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:25,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:25,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653865994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:25,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:25,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:25,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:25,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:25,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:25,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653865994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:25,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:25,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:25,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T20:43:26,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/6baa337f0db14c7bbdfaeed4d0f4b3f7 2024-11-26T20:43:26,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/b8367dd515e34bea9404f0c0f1f0c9c7 is 50, key is test_row_0/C:col10/1732653804835/Put/seqid=0 2024-11-26T20:43:26,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742405_1581 (size=12151) 2024-11-26T20:43:26,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T20:43:26,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:26,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:26,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:26,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:26,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:26,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T20:43:26,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:26,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:26,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:26,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:26,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:26,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T20:43:26,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:26,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:26,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:26,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:26,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:26,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/b8367dd515e34bea9404f0c0f1f0c9c7 2024-11-26T20:43:26,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/df84956101094be3b344331dc83f3d12 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/df84956101094be3b344331dc83f3d12 2024-11-26T20:43:26,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/df84956101094be3b344331dc83f3d12, entries=200, sequenceid=224, filesize=38.8 K 2024-11-26T20:43:26,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/6baa337f0db14c7bbdfaeed4d0f4b3f7 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/6baa337f0db14c7bbdfaeed4d0f4b3f7 2024-11-26T20:43:26,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/6baa337f0db14c7bbdfaeed4d0f4b3f7, entries=150, sequenceid=224, filesize=11.9 K 2024-11-26T20:43:26,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/b8367dd515e34bea9404f0c0f1f0c9c7 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/b8367dd515e34bea9404f0c0f1f0c9c7 2024-11-26T20:43:26,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/b8367dd515e34bea9404f0c0f1f0c9c7, entries=150, sequenceid=224, filesize=11.9 K 2024-11-26T20:43:26,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 0c644311c9706b1426d8809c603e0fe8 in 1655ms, sequenceid=224, compaction requested=false 2024-11-26T20:43:26,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:26,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T20:43:26,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:26,559 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 0c644311c9706b1426d8809c603e0fe8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T20:43:26,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=A 2024-11-26T20:43:26,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:26,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=B 2024-11-26T20:43:26,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:26,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=C 2024-11-26T20:43:26,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:26,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260ade8d35a7ee429ba5948b507c9eb30d_0c644311c9706b1426d8809c603e0fe8 is 50, key is test_row_0/A:col10/1732653804849/Put/seqid=0 2024-11-26T20:43:26,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742406_1582 (size=12304) 2024-11-26T20:43:26,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:26,585 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260ade8d35a7ee429ba5948b507c9eb30d_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260ade8d35a7ee429ba5948b507c9eb30d_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:26,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/161c4b216cf543f69e54017b9be8bb20, store: [table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:26,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/161c4b216cf543f69e54017b9be8bb20 is 175, key is test_row_0/A:col10/1732653804849/Put/seqid=0 2024-11-26T20:43:26,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742407_1583 (size=31105) 2024-11-26T20:43:26,990 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/161c4b216cf543f69e54017b9be8bb20 2024-11-26T20:43:26,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/3bd89dfdca264a34982863a4bea649a4 is 50, key is test_row_0/B:col10/1732653804849/Put/seqid=0 2024-11-26T20:43:27,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742408_1584 (size=12151) 2024-11-26T20:43:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:27,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T20:43:27,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653867041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:27,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653807029, max timestamp=1732653807045Num uniques 859; 2024-11-26T20:43:27,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:43:27,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653807029, max timestamp=1732653807045Num uniques 859; 2024-11-26T20:43:27,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:43:27,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732653807029, max timestamp=1732653807045Num uniques 859; 2024-11-26T20:43:27,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T20:43:27,049 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,049 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,049 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:27,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653867093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:27,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:27,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653867097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:27,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:27,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653867154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:27,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:27,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653867204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:27,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:27,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653867214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:27,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:27,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653867364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:27,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:27,401 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/3bd89dfdca264a34982863a4bea649a4 2024-11-26T20:43:27,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/3fe944a345e44de793f3cf119230d2c3 is 50, key is test_row_0/C:col10/1732653804849/Put/seqid=0 2024-11-26T20:43:27,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742409_1585 (size=12151) 2024-11-26T20:43:27,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653867414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:27,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:27,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653867424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:27,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:27,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:27,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653867674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:27,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:27,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653867723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:27,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:27,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653867733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:27,811 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/3fe944a345e44de793f3cf119230d2c3 2024-11-26T20:43:27,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/161c4b216cf543f69e54017b9be8bb20 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/161c4b216cf543f69e54017b9be8bb20 2024-11-26T20:43:27,818 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/161c4b216cf543f69e54017b9be8bb20, entries=150, sequenceid=236, filesize=30.4 K 2024-11-26T20:43:27,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/3bd89dfdca264a34982863a4bea649a4 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/3bd89dfdca264a34982863a4bea649a4 2024-11-26T20:43:27,821 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/3bd89dfdca264a34982863a4bea649a4, entries=150, sequenceid=236, filesize=11.9 K 2024-11-26T20:43:27,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/3fe944a345e44de793f3cf119230d2c3 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/3fe944a345e44de793f3cf119230d2c3 2024-11-26T20:43:27,825 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/3fe944a345e44de793f3cf119230d2c3, entries=150, sequenceid=236, filesize=11.9 K 2024-11-26T20:43:27,825 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 0c644311c9706b1426d8809c603e0fe8 in 1266ms, sequenceid=236, compaction requested=true 2024-11-26T20:43:27,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:27,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:27,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-26T20:43:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-26T20:43:27,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-26T20:43:27,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9510 sec 2024-11-26T20:43:27,830 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.9550 sec 2024-11-26T20:43:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:28,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0c644311c9706b1426d8809c603e0fe8 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-26T20:43:28,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=A 2024-11-26T20:43:28,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:43:28,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=B 2024-11-26T20:43:28,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:43:28,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=C 2024-11-26T20:43:28,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:43:28,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112692b8d340d1e84a859733acc893f3d403_0c644311c9706b1426d8809c603e0fe8 is 50, key is test_row_0/A:col10/1732653808195/Put/seqid=0 2024-11-26T20:43:28,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742410_1586 (size=12454) 2024-11-26T20:43:28,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:28,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653868233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:28,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:28,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:28,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:28,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:28,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653868234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:28,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:28,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:28,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:28,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:28,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653868243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:28,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:28,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:28,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:28,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653868344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:28,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:28,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:28,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:28,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653868554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:28,608 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,612 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112692b8d340d1e84a859733acc893f3d403_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112692b8d340d1e84a859733acc893f3d403_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:28,613 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/260687fe68c445a095bfd9579f8d53c4, store: [table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:28,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/260687fe68c445a095bfd9579f8d53c4 is 175, key is test_row_0/A:col10/1732653808195/Put/seqid=0 2024-11-26T20:43:28,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742411_1587 (size=31255) 2024-11-26T20:43:28,621 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/260687fe68c445a095bfd9579f8d53c4 2024-11-26T20:43:28,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/1b90b686febc47bbb49d0c3068a9e362 is 50, key is test_row_0/B:col10/1732653808195/Put/seqid=0 2024-11-26T20:43:28,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742412_1588 (size=12301) 2024-11-26T20:43:28,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/1b90b686febc47bbb49d0c3068a9e362 2024-11-26T20:43:28,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/c4402ec8c6654bdaa48827bc1cc6a086 is 50, key is test_row_0/C:col10/1732653808195/Put/seqid=0 2024-11-26T20:43:28,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742413_1589 (size=12301) 2024-11-26T20:43:28,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/c4402ec8c6654bdaa48827bc1cc6a086 2024-11-26T20:43:28,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/260687fe68c445a095bfd9579f8d53c4 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/260687fe68c445a095bfd9579f8d53c4 2024-11-26T20:43:28,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/260687fe68c445a095bfd9579f8d53c4, entries=150, sequenceid=272, filesize=30.5 K 2024-11-26T20:43:28,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/1b90b686febc47bbb49d0c3068a9e362 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/1b90b686febc47bbb49d0c3068a9e362 2024-11-26T20:43:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/1b90b686febc47bbb49d0c3068a9e362, entries=150, sequenceid=272, filesize=12.0 K 2024-11-26T20:43:28,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/c4402ec8c6654bdaa48827bc1cc6a086 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/c4402ec8c6654bdaa48827bc1cc6a086 2024-11-26T20:43:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/c4402ec8c6654bdaa48827bc1cc6a086, entries=150, sequenceid=272, filesize=12.0 K 2024-11-26T20:43:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 0c644311c9706b1426d8809c603e0fe8 in 515ms, sequenceid=272, compaction requested=true 2024-11-26T20:43:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:28,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0c644311c9706b1426d8809c603e0fe8:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:43:28,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:28,713 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:43:28,713 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:43:28,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0c644311c9706b1426d8809c603e0fe8:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:43:28,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:28,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0c644311c9706b1426d8809c603e0fe8:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:43:28,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:43:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,714 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:43:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,714 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133658 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:43:28,714 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 0c644311c9706b1426d8809c603e0fe8/A is initiating minor compaction (all files) 2024-11-26T20:43:28,714 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 0c644311c9706b1426d8809c603e0fe8/B is initiating minor compaction (all files) 2024-11-26T20:43:28,714 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0c644311c9706b1426d8809c603e0fe8/B in TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:28,714 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0c644311c9706b1426d8809c603e0fe8/A in TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:28,714 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/9ea2dc884f99456c996e798660427ece, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/6baa337f0db14c7bbdfaeed4d0f4b3f7, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/3bd89dfdca264a34982863a4bea649a4, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/1b90b686febc47bbb49d0c3068a9e362] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp, totalSize=48.0 K 2024-11-26T20:43:28,714 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/26ae3b70f2ab49c0ae9c04d179278741, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/df84956101094be3b344331dc83f3d12, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/161c4b216cf543f69e54017b9be8bb20, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/260687fe68c445a095bfd9579f8d53c4] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp, totalSize=130.5 K 2024-11-26T20:43:28,714 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:28,714 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. files: [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/26ae3b70f2ab49c0ae9c04d179278741, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/df84956101094be3b344331dc83f3d12, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/161c4b216cf543f69e54017b9be8bb20, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/260687fe68c445a095bfd9579f8d53c4] 2024-11-26T20:43:28,715 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26ae3b70f2ab49c0ae9c04d179278741, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732653803019 2024-11-26T20:43:28,715 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ea2dc884f99456c996e798660427ece, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732653803019 2024-11-26T20:43:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,716 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting df84956101094be3b344331dc83f3d12, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732653804194 2024-11-26T20:43:28,716 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 6baa337f0db14c7bbdfaeed4d0f4b3f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732653804194 2024-11-26T20:43:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,716 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 161c4b216cf543f69e54017b9be8bb20, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732653804849 2024-11-26T20:43:28,716 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bd89dfdca264a34982863a4bea649a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732653804849 2024-11-26T20:43:28,716 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b90b686febc47bbb49d0c3068a9e362, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732653807089 2024-11-26T20:43:28,716 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 260687fe68c445a095bfd9579f8d53c4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732653807089 2024-11-26T20:43:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,725 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,726 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0c644311c9706b1426d8809c603e0fe8#B#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:43:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,727 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/a7282634c0a24671b8407e7c3efb9175 is 50, key is test_row_0/B:col10/1732653808195/Put/seqid=0 2024-11-26T20:43:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,728 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112612d2e458a6be499a999a3f5880c73622_0c644311c9706b1426d8809c603e0fe8 store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:28,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742414_1590 (size=12881) 2024-11-26T20:43:28,731 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112612d2e458a6be499a999a3f5880c73622_0c644311c9706b1426d8809c603e0fe8, store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:28,731 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112612d2e458a6be499a999a3f5880c73622_0c644311c9706b1426d8809c603e0fe8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742415_1591 (size=4469) 2024-11-26T20:43:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,739 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0c644311c9706b1426d8809c603e0fe8#A#compaction#502 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:43:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,740 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/0b7039fcc27d4dee93f95143d3407067 is 175, key is test_row_0/A:col10/1732653808195/Put/seqid=0 2024-11-26T20:43:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742416_1592 (size=31835) 2024-11-26T20:43:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0c644311c9706b1426d8809c603e0fe8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T20:43:28,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=A 2024-11-26T20:43:28,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:28,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=B 2024-11-26T20:43:28,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:28,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=C 2024-11-26T20:43:28,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126714cf80d41874bf8bd42a1b3240cba06_0c644311c9706b1426d8809c603e0fe8 is 50, key is test_row_0/A:col10/1732653808231/Put/seqid=0 2024-11-26T20:43:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742417_1593 (size=12454) 2024-11-26T20:43:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,881 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,885 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126714cf80d41874bf8bd42a1b3240cba06_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126714cf80d41874bf8bd42a1b3240cba06_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:28,885 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/a34e952f70a9420faeed357a87eb4304, store: [table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:28,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/a34e952f70a9420faeed357a87eb4304 is 175, key is test_row_0/A:col10/1732653808231/Put/seqid=0 2024-11-26T20:43:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742418_1594 (size=31251) 2024-11-26T20:43:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:28,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653808952, max timestamp=1732653808974Num uniques 859; 2024-11-26T20:43:28,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:43:28,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653808952, max timestamp=1732653808974Num uniques 859; 2024-11-26T20:43:28,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732653808952, max timestamp=1732653808974Num uniques 859; 2024-11-26T20:43:28,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:43:28,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cd6be161543e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T20:43:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T20:43:29,034 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:43:29,035 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:43:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-26T20:43:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T20:43:29,036 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:43:29,037 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:43:29,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:43:29,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653869068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:29,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46164 deadline: 1732653869093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:29,098 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 0c644311c9706b1426d8809c603e0fe8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:43:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:29,135 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/a7282634c0a24671b8407e7c3efb9175 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/a7282634c0a24671b8407e7c3efb9175 2024-11-26T20:43:29,138 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0c644311c9706b1426d8809c603e0fe8/B of 0c644311c9706b1426d8809c603e0fe8 into a7282634c0a24671b8407e7c3efb9175(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:43:29,139 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:29,139 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., storeName=0c644311c9706b1426d8809c603e0fe8/B, priority=12, startTime=1732653808713; duration=0sec 2024-11-26T20:43:29,139 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:43:29,139 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0c644311c9706b1426d8809c603e0fe8:B 2024-11-26T20:43:29,139 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:43:29,140 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:43:29,140 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 0c644311c9706b1426d8809c603e0fe8/C is initiating minor compaction (all files) 2024-11-26T20:43:29,140 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0c644311c9706b1426d8809c603e0fe8/C in TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:29,140 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/528123e9fc804c82b16ca8c03809cb61, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/b8367dd515e34bea9404f0c0f1f0c9c7, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/3fe944a345e44de793f3cf119230d2c3, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/c4402ec8c6654bdaa48827bc1cc6a086] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp, totalSize=48.0 K 2024-11-26T20:43:29,140 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 528123e9fc804c82b16ca8c03809cb61, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732653803019 2024-11-26T20:43:29,140 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting b8367dd515e34bea9404f0c0f1f0c9c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732653804194 2024-11-26T20:43:29,141 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fe944a345e44de793f3cf119230d2c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732653804849 2024-11-26T20:43:29,141 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting c4402ec8c6654bdaa48827bc1cc6a086, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732653807089 2024-11-26T20:43:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T20:43:29,154 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/0b7039fcc27d4dee93f95143d3407067 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/0b7039fcc27d4dee93f95143d3407067 2024-11-26T20:43:29,155 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0c644311c9706b1426d8809c603e0fe8#C#compaction#504 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:43:29,156 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/785876f9e97f42e4ba5cc3e353d25437 is 50, key is test_row_0/C:col10/1732653808195/Put/seqid=0 2024-11-26T20:43:29,158 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0c644311c9706b1426d8809c603e0fe8/A of 0c644311c9706b1426d8809c603e0fe8 into 0b7039fcc27d4dee93f95143d3407067(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:43:29,158 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:29,158 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., storeName=0c644311c9706b1426d8809c603e0fe8/A, priority=12, startTime=1732653808713; duration=0sec 2024-11-26T20:43:29,158 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:29,158 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0c644311c9706b1426d8809c603e0fe8:A 2024-11-26T20:43:29,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742419_1595 (size=12881) 2024-11-26T20:43:29,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653869184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:29,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:29,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:29,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:29,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-26T20:43:29,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:29,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:29,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:29,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:29,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:29,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653869254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:29,258 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_0' in 0c644311c9706b1426d8809c603e0fe8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:43:29,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:29,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:29,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:29,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653869264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:29,265 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_2' in 0c644311c9706b1426d8809c603e0fe8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:43:29,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:29,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:29,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:29,290 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=283, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/a34e952f70a9420faeed357a87eb4304 2024-11-26T20:43:29,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/ccd7cefc98d244dcbfb65cfe6e75ed55 is 50, key is test_row_0/B:col10/1732653808231/Put/seqid=0 2024-11-26T20:43:29,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742420_1596 (size=9857) 2024-11-26T20:43:29,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/ccd7cefc98d244dcbfb65cfe6e75ed55 2024-11-26T20:43:29,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/e8783c5674b341a09de0ed9bbe590b43 is 50, key is test_row_0/C:col10/1732653808231/Put/seqid=0 2024-11-26T20:43:29,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742421_1597 (size=9857) 2024-11-26T20:43:29,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/e8783c5674b341a09de0ed9bbe590b43 2024-11-26T20:43:29,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/a34e952f70a9420faeed357a87eb4304 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/a34e952f70a9420faeed357a87eb4304 2024-11-26T20:43:29,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/a34e952f70a9420faeed357a87eb4304, entries=150, sequenceid=283, filesize=30.5 K 2024-11-26T20:43:29,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/ccd7cefc98d244dcbfb65cfe6e75ed55 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/ccd7cefc98d244dcbfb65cfe6e75ed55 2024-11-26T20:43:29,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/ccd7cefc98d244dcbfb65cfe6e75ed55, entries=100, sequenceid=283, filesize=9.6 K 2024-11-26T20:43:29,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/e8783c5674b341a09de0ed9bbe590b43 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/e8783c5674b341a09de0ed9bbe590b43 2024-11-26T20:43:29,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/e8783c5674b341a09de0ed9bbe590b43, entries=100, sequenceid=283, filesize=9.6 K 2024-11-26T20:43:29,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 0c644311c9706b1426d8809c603e0fe8 in 458ms, sequenceid=283, compaction requested=false 2024-11-26T20:43:29,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:29,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-26T20:43:29,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:29,342 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 0c644311c9706b1426d8809c603e0fe8 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-26T20:43:29,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=A 2024-11-26T20:43:29,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:43:29,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=B 2024-11-26T20:43:29,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:43:29,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=C 2024-11-26T20:43:29,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T20:43:29,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126eb2f28a0b633498383f42bc6b991fae6_0c644311c9706b1426d8809c603e0fe8 is 50, key is test_row_0/A:col10/1732653809066/Put/seqid=0 2024-11-26T20:43:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T20:43:29,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742422_1598 (size=12454) 2024-11-26T20:43:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:29,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:29,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653869436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:29,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:29,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:29,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:29,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653869544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:29,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:29,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:29,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:29,565 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/785876f9e97f42e4ba5cc3e353d25437 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/785876f9e97f42e4ba5cc3e353d25437 2024-11-26T20:43:29,569 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0c644311c9706b1426d8809c603e0fe8/C of 0c644311c9706b1426d8809c603e0fe8 into 785876f9e97f42e4ba5cc3e353d25437(size=12.6 K), total size for store is 22.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:43:29,570 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:29,570 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., storeName=0c644311c9706b1426d8809c603e0fe8/C, priority=12, startTime=1732653808713; duration=0sec 2024-11-26T20:43:29,570 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:29,570 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0c644311c9706b1426d8809c603e0fe8:C 2024-11-26T20:43:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T20:43:29,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653869754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:29,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:29,763 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126eb2f28a0b633498383f42bc6b991fae6_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126eb2f28a0b633498383f42bc6b991fae6_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:29,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/ee166d47f95c45bdad4abd8f738440b9, store: [table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:29,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/ee166d47f95c45bdad4abd8f738440b9 is 175, key is test_row_0/A:col10/1732653809066/Put/seqid=0 2024-11-26T20:43:29,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742423_1599 (size=31255) 2024-11-26T20:43:29,967 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T20:43:30,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653870064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:30,169 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/ee166d47f95c45bdad4abd8f738440b9 2024-11-26T20:43:30,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T20:43:30,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/4067eb7e77b94e4087f9cc1599d2c7d0 is 50, key is test_row_0/B:col10/1732653809066/Put/seqid=0 2024-11-26T20:43:30,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742424_1600 (size=12301) 2024-11-26T20:43:30,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653870574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:30,576 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:30,576 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:30,576 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:30,579 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/4067eb7e77b94e4087f9cc1599d2c7d0 2024-11-26T20:43:30,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/edb9dfa020df4321b0dde08d963de656 is 50, key is test_row_0/C:col10/1732653809066/Put/seqid=0 2024-11-26T20:43:30,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742425_1601 (size=12301) 2024-11-26T20:43:30,589 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/edb9dfa020df4321b0dde08d963de656 2024-11-26T20:43:30,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/ee166d47f95c45bdad4abd8f738440b9 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/ee166d47f95c45bdad4abd8f738440b9 2024-11-26T20:43:30,596 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/ee166d47f95c45bdad4abd8f738440b9, entries=150, sequenceid=321, filesize=30.5 K 2024-11-26T20:43:30,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/4067eb7e77b94e4087f9cc1599d2c7d0 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/4067eb7e77b94e4087f9cc1599d2c7d0 2024-11-26T20:43:30,600 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/4067eb7e77b94e4087f9cc1599d2c7d0, entries=150, sequenceid=321, filesize=12.0 K 2024-11-26T20:43:30,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/edb9dfa020df4321b0dde08d963de656 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/edb9dfa020df4321b0dde08d963de656 2024-11-26T20:43:30,605 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/edb9dfa020df4321b0dde08d963de656, entries=150, sequenceid=321, filesize=12.0 K 2024-11-26T20:43:30,606 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 0c644311c9706b1426d8809c603e0fe8 in 1264ms, sequenceid=321, compaction requested=true 2024-11-26T20:43:30,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:30,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:30,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-26T20:43:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-26T20:43:30,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-26T20:43:30,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5700 sec 2024-11-26T20:43:30,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.5740 sec 2024-11-26T20:43:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T20:43:31,184 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:43:31,185 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T20:43:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-26T20:43:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-26T20:43:31,187 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T20:43:31,187 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T20:43:31,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T20:43:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:31,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0c644311c9706b1426d8809c603e0fe8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T20:43:31,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=A 2024-11-26T20:43:31,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:31,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=B 2024-11-26T20:43:31,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:31,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=C 2024-11-26T20:43:31,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:31,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126aeff7e8911364f799c82faa4fe34293e_0c644311c9706b1426d8809c603e0fe8 is 50, key is test_row_0/A:col10/1732653811264/Put/seqid=0 2024-11-26T20:43:31,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742426_1602 (size=12454) 2024-11-26T20:43:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-26T20:43:31,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-26T20:43:31,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:31,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:31,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:31,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:31,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:31,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653871338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:31,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:31,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:31,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:31,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653871340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:31,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:31,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:31,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:31,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1732653871443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:31,448 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_2' in 0c644311c9706b1426d8809c603e0fe8 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:43:31,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:31,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:31,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:31,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653871454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:31,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653871454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:31,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:31,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:31,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:31,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:31,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:31,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:31,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-26T20:43:31,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:31,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:31,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:31,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:31,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-26T20:43:31,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732653871583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:31,586 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to cd6be161543e,40291,1732653631795 for 'test_row_1' in 0c644311c9706b1426d8809c603e0fe8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T20:43:31,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:31,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:31,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:31,618 INFO [Thread-2348 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2348. 2024-11-26T20:43:31,618 DEBUG [Thread-2348 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:43:31,618 DEBUG [Thread-2348 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:31,618 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:43:31,618 DEBUG [Thread-2348 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:31,620 INFO [Thread-2346 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2346. 2024-11-26T20:43:31,621 DEBUG [Thread-2346 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:43:31,621 DEBUG [Thread-2346 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:31,621 DEBUG [Thread-2346 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:31,621 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:43:31,623 INFO [Thread-2352 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2352. 2024-11-26T20:43:31,623 DEBUG [Thread-2352 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:43:31,623 DEBUG [Thread-2352 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:31,623 DEBUG [Thread-2352 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:31,623 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:43:31,626 INFO [Thread-2344 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2344. 2024-11-26T20:43:31,626 DEBUG [Thread-2344 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:43:31,626 DEBUG [Thread-2344 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:31,626 DEBUG [Thread-2344 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:31,626 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:43:31,627 INFO [Thread-2350 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2350. 2024-11-26T20:43:31,627 DEBUG [Thread-2350 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:43:31,627 DEBUG [Thread-2350 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:31,627 DEBUG [Thread-2350 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:31,627 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:43:31,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-26T20:43:31,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:31,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:31,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:31,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:31,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:31,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:31,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653871664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653871664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:31,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:31,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:31,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:31,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:31,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:31,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:31,676 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:31,679 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126aeff7e8911364f799c82faa4fe34293e_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126aeff7e8911364f799c82faa4fe34293e_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:31,679 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/05c524de51e64d0db3e897f35d8a0bf1, store: [table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:31,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/05c524de51e64d0db3e897f35d8a0bf1 is 175, key is test_row_0/A:col10/1732653811264/Put/seqid=0 2024-11-26T20:43:31,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742427_1603 (size=31255) 2024-11-26T20:43:31,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-26T20:43:31,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:31,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:31,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:31,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:31,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:31,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:31,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-26T20:43:31,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-26T20:43:31,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:31,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:31,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:31,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:31,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:31,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:31,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:31,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:31,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653871974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:31,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653871974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:31,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:31,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:31,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:31,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:31,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:31,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:32,083 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/05c524de51e64d0db3e897f35d8a0bf1 2024-11-26T20:43:32,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/3d1dcd729acf4c21b6687edbbd8584e8 is 50, key is test_row_0/B:col10/1732653811264/Put/seqid=0 2024-11-26T20:43:32,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742428_1604 (size=12301) 2024-11-26T20:43:32,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-26T20:43:32,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:32,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:32,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:32,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:32,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:32,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:32,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-26T20:43:32,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:32,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:32,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:32,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:32,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:32,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:32,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-26T20:43:32,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-26T20:43:32,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:32,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:32,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:32,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:32,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:43:32,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:32,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46148 deadline: 1732653872484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:32,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T20:43:32,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46142 deadline: 1732653872484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 2024-11-26T20:43:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 , the old value is region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T20:43:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., hostname=cd6be161543e,40291,1732653631795, seqNum=5 because the exception is null or not the one we care about 2024-11-26T20:43:32,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/3d1dcd729acf4c21b6687edbbd8584e8 2024-11-26T20:43:32,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/fd587b0ab51f447f93b5f3def5a51a66 is 50, key is test_row_0/C:col10/1732653811264/Put/seqid=0 2024-11-26T20:43:32,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742429_1605 (size=12301) 2024-11-26T20:43:32,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/fd587b0ab51f447f93b5f3def5a51a66 2024-11-26T20:43:32,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/05c524de51e64d0db3e897f35d8a0bf1 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/05c524de51e64d0db3e897f35d8a0bf1 2024-11-26T20:43:32,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/05c524de51e64d0db3e897f35d8a0bf1, entries=150, sequenceid=334, filesize=30.5 K 2024-11-26T20:43:32,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/3d1dcd729acf4c21b6687edbbd8584e8 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/3d1dcd729acf4c21b6687edbbd8584e8 2024-11-26T20:43:32,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/3d1dcd729acf4c21b6687edbbd8584e8, entries=150, sequenceid=334, filesize=12.0 K 2024-11-26T20:43:32,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/fd587b0ab51f447f93b5f3def5a51a66 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/fd587b0ab51f447f93b5f3def5a51a66 2024-11-26T20:43:32,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/fd587b0ab51f447f93b5f3def5a51a66, entries=150, sequenceid=334, filesize=12.0 K 2024-11-26T20:43:32,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0c644311c9706b1426d8809c603e0fe8 in 1270ms, sequenceid=334, compaction requested=true 2024-11-26T20:43:32,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:32,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0c644311c9706b1426d8809c603e0fe8:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T20:43:32,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:32,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0c644311c9706b1426d8809c603e0fe8:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T20:43:32,536 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:43:32,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:32,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0c644311c9706b1426d8809c603e0fe8:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T20:43:32,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:43:32,536 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:43:32,537 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47340 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:43:32,537 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125596 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:43:32,537 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 0c644311c9706b1426d8809c603e0fe8/B is initiating minor compaction (all files) 2024-11-26T20:43:32,537 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1541): 0c644311c9706b1426d8809c603e0fe8/A is initiating minor compaction (all files) 2024-11-26T20:43:32,537 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0c644311c9706b1426d8809c603e0fe8/B in TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:32,537 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0c644311c9706b1426d8809c603e0fe8/A in TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:32,537 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/a7282634c0a24671b8407e7c3efb9175, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/ccd7cefc98d244dcbfb65cfe6e75ed55, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/4067eb7e77b94e4087f9cc1599d2c7d0, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/3d1dcd729acf4c21b6687edbbd8584e8] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp, totalSize=46.2 K 2024-11-26T20:43:32,537 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/0b7039fcc27d4dee93f95143d3407067, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/a34e952f70a9420faeed357a87eb4304, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/ee166d47f95c45bdad4abd8f738440b9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/05c524de51e64d0db3e897f35d8a0bf1] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp, totalSize=122.7 K 2024-11-26T20:43:32,537 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:32,537 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. files: [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/0b7039fcc27d4dee93f95143d3407067, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/a34e952f70a9420faeed357a87eb4304, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/ee166d47f95c45bdad4abd8f738440b9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/05c524de51e64d0db3e897f35d8a0bf1] 2024-11-26T20:43:32,538 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting a7282634c0a24671b8407e7c3efb9175, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732653807089 2024-11-26T20:43:32,538 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b7039fcc27d4dee93f95143d3407067, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732653807089 2024-11-26T20:43:32,538 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting ccd7cefc98d244dcbfb65cfe6e75ed55, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732653808231 2024-11-26T20:43:32,538 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting a34e952f70a9420faeed357a87eb4304, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732653808226 2024-11-26T20:43:32,538 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 4067eb7e77b94e4087f9cc1599d2c7d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732653809039 2024-11-26T20:43:32,538 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee166d47f95c45bdad4abd8f738440b9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732653809039 2024-11-26T20:43:32,538 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d1dcd729acf4c21b6687edbbd8584e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732653809412 2024-11-26T20:43:32,538 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05c524de51e64d0db3e897f35d8a0bf1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732653809412 2024-11-26T20:43:32,545 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:32,546 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0c644311c9706b1426d8809c603e0fe8#B#compaction#513 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:43:32,546 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/1caf48edf2bc49629f7edb8f3b211b32 is 50, key is test_row_0/B:col10/1732653811264/Put/seqid=0 2024-11-26T20:43:32,547 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411269a4477dcbbcf4174a31d0854c7167dac_0c644311c9706b1426d8809c603e0fe8 store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:32,551 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411269a4477dcbbcf4174a31d0854c7167dac_0c644311c9706b1426d8809c603e0fe8, store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:32,551 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411269a4477dcbbcf4174a31d0854c7167dac_0c644311c9706b1426d8809c603e0fe8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:32,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40291 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-26T20:43:32,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:32,564 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 0c644311c9706b1426d8809c603e0fe8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T20:43:32,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=A 2024-11-26T20:43:32,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:32,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=B 2024-11-26T20:43:32,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:32,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=C 2024-11-26T20:43:32,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:32,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742430_1606 (size=13017) 2024-11-26T20:43:32,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742431_1607 (size=4469) 2024-11-26T20:43:32,568 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0c644311c9706b1426d8809c603e0fe8#A#compaction#514 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:43:32,568 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/b55107f6795b46cca3aed06b64f0b815 is 175, key is test_row_0/A:col10/1732653811264/Put/seqid=0 2024-11-26T20:43:32,571 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/1caf48edf2bc49629f7edb8f3b211b32 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/1caf48edf2bc49629f7edb8f3b211b32 2024-11-26T20:43:32,575 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0c644311c9706b1426d8809c603e0fe8/B of 0c644311c9706b1426d8809c603e0fe8 into 1caf48edf2bc49629f7edb8f3b211b32(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:43:32,575 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:32,575 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., storeName=0c644311c9706b1426d8809c603e0fe8/B, priority=12, startTime=1732653812536; duration=0sec 2024-11-26T20:43:32,575 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T20:43:32,575 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0c644311c9706b1426d8809c603e0fe8:B 2024-11-26T20:43:32,575 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T20:43:32,576 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47340 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T20:43:32,577 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1541): 0c644311c9706b1426d8809c603e0fe8/C is initiating minor compaction (all files) 2024-11-26T20:43:32,577 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0c644311c9706b1426d8809c603e0fe8/C in TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:32,577 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/785876f9e97f42e4ba5cc3e353d25437, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/e8783c5674b341a09de0ed9bbe590b43, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/edb9dfa020df4321b0dde08d963de656, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/fd587b0ab51f447f93b5f3def5a51a66] into tmpdir=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp, totalSize=46.2 K 2024-11-26T20:43:32,577 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting 785876f9e97f42e4ba5cc3e353d25437, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732653807089 2024-11-26T20:43:32,577 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting e8783c5674b341a09de0ed9bbe590b43, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732653808231 2024-11-26T20:43:32,577 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting edb9dfa020df4321b0dde08d963de656, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732653809039 2024-11-26T20:43:32,578 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] compactions.Compactor(225): Compacting fd587b0ab51f447f93b5f3def5a51a66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732653809412 2024-11-26T20:43:32,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126bd2dce9d307f40c291d5fd3cf3486b6d_0c644311c9706b1426d8809c603e0fe8 is 50, key is test_row_0/A:col10/1732653811332/Put/seqid=0 2024-11-26T20:43:32,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742432_1608 (size=31971) 2024-11-26T20:43:32,593 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0c644311c9706b1426d8809c603e0fe8#C#compaction#516 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T20:43:32,593 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/bef3498e4deb4986a81dfea48e06af01 is 50, key is test_row_0/C:col10/1732653811264/Put/seqid=0 2024-11-26T20:43:32,594 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/b55107f6795b46cca3aed06b64f0b815 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/b55107f6795b46cca3aed06b64f0b815 2024-11-26T20:43:32,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742433_1609 (size=12454) 2024-11-26T20:43:32,598 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0c644311c9706b1426d8809c603e0fe8/A of 0c644311c9706b1426d8809c603e0fe8 into b55107f6795b46cca3aed06b64f0b815(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:43:32,598 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:32,598 INFO [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., storeName=0c644311c9706b1426d8809c603e0fe8/A, priority=12, startTime=1732653812536; duration=0sec 2024-11-26T20:43:32,598 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:32,598 DEBUG [RS:0;cd6be161543e:40291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0c644311c9706b1426d8809c603e0fe8:A 2024-11-26T20:43:32,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742434_1610 (size=13017) 2024-11-26T20:43:32,609 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/bef3498e4deb4986a81dfea48e06af01 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/bef3498e4deb4986a81dfea48e06af01 2024-11-26T20:43:32,613 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0c644311c9706b1426d8809c603e0fe8/C of 0c644311c9706b1426d8809c603e0fe8 into bef3498e4deb4986a81dfea48e06af01(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T20:43:32,613 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:32,613 INFO [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., storeName=0c644311c9706b1426d8809c603e0fe8/C, priority=12, startTime=1732653812536; duration=0sec 2024-11-26T20:43:32,613 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T20:43:32,613 DEBUG [RS:0;cd6be161543e:40291-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0c644311c9706b1426d8809c603e0fe8:C 2024-11-26T20:43:32,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:33,004 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126bd2dce9d307f40c291d5fd3cf3486b6d_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126bd2dce9d307f40c291d5fd3cf3486b6d_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:33,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/1dde5aa80874440f835efbb918375e86, store: [table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:33,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/1dde5aa80874440f835efbb918375e86 is 175, key is test_row_0/A:col10/1732653811332/Put/seqid=0 2024-11-26T20:43:33,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742435_1611 (size=31255) 2024-11-26T20:43:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-26T20:43:33,410 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/1dde5aa80874440f835efbb918375e86 2024-11-26T20:43:33,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/985456ce31a641adb4e5d9d36ea2cd50 is 50, key is test_row_0/B:col10/1732653811332/Put/seqid=0 2024-11-26T20:43:33,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742436_1612 (size=12301) 2024-11-26T20:43:33,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40291 {}] regionserver.HRegion(8855): Flush requested on 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:33,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. as already flushing 2024-11-26T20:43:33,495 INFO [Thread-2341 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2341. 2024-11-26T20:43:33,495 DEBUG [Thread-2341 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:43:33,495 DEBUG [Thread-2341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:33,495 DEBUG [Thread-2341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:33,495 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:43:33,505 INFO [Thread-2333 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2333. 2024-11-26T20:43:33,505 DEBUG [Thread-2333 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:43:33,505 DEBUG [Thread-2333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:33,505 DEBUG [Thread-2333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:33,505 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:43:33,595 INFO [Thread-2335 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2335. 2024-11-26T20:43:33,595 DEBUG [Thread-2335 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:43:33,595 DEBUG [Thread-2335 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:33,595 DEBUG [Thread-2335 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:33,595 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:43:33,819 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/985456ce31a641adb4e5d9d36ea2cd50 2024-11-26T20:43:33,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/389c91cfefc94dc28ef7bcfce9908a79 is 50, key is test_row_0/C:col10/1732653811332/Put/seqid=0 2024-11-26T20:43:33,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742437_1613 (size=12301) 2024-11-26T20:43:34,228 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/389c91cfefc94dc28ef7bcfce9908a79 2024-11-26T20:43:34,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/1dde5aa80874440f835efbb918375e86 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/1dde5aa80874440f835efbb918375e86 2024-11-26T20:43:34,233 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/1dde5aa80874440f835efbb918375e86, entries=150, sequenceid=354, filesize=30.5 K 2024-11-26T20:43:34,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/985456ce31a641adb4e5d9d36ea2cd50 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/985456ce31a641adb4e5d9d36ea2cd50 2024-11-26T20:43:34,236 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/985456ce31a641adb4e5d9d36ea2cd50, entries=150, sequenceid=354, filesize=12.0 K 2024-11-26T20:43:34,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/389c91cfefc94dc28ef7bcfce9908a79 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/389c91cfefc94dc28ef7bcfce9908a79 2024-11-26T20:43:34,239 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/389c91cfefc94dc28ef7bcfce9908a79, entries=150, sequenceid=354, filesize=12.0 K 2024-11-26T20:43:34,240 INFO [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=20.13 KB/20610 for 0c644311c9706b1426d8809c603e0fe8 in 1676ms, sequenceid=354, compaction requested=false 2024-11-26T20:43:34,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 0c644311c9706b1426d8809c603e0fe8: 2024-11-26T20:43:34,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:34,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cd6be161543e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-26T20:43:34,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-26T20:43:34,242 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-26T20:43:34,242 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0540 sec 2024-11-26T20:43:34,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 3.0580 sec 2024-11-26T20:43:35,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-26T20:43:35,364 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:43:39,145 INFO [Thread-2339 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2339. 2024-11-26T20:43:39,145 DEBUG [Thread-2339 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:43:39,145 DEBUG [Thread-2339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:39,145 DEBUG [Thread-2339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:39,145 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:43:41,515 INFO [Thread-2337 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2337. 2024-11-26T20:43:41,515 DEBUG [Thread-2337 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T20:43:41,515 DEBUG [Thread-2337 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:41,515 DEBUG [Thread-2337 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:41,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:43:41,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T20:43:41,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-26T20:43:41,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-26T20:43:41,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-26T20:43:41,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-26T20:43:41,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-26T20:43:41,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T20:43:41,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3109 2024-11-26T20:43:41,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2975 2024-11-26T20:43:41,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2955 2024-11-26T20:43:41,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3116 2024-11-26T20:43:41,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2966 2024-11-26T20:43:41,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T20:43:41,516 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T20:43:41,516 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:43:41,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:41,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:41,516 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:43:41,517 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T20:43:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T20:43:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-26T20:43:41,519 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653821519"}]},"ts":"1732653821519"} 2024-11-26T20:43:41,520 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T20:43:41,520 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T20:43:41,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T20:43:41,522 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0c644311c9706b1426d8809c603e0fe8, UNASSIGN}] 2024-11-26T20:43:41,522 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0c644311c9706b1426d8809c603e0fe8, UNASSIGN 2024-11-26T20:43:41,523 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=0c644311c9706b1426d8809c603e0fe8, regionState=CLOSING, regionLocation=cd6be161543e,40291,1732653631795 2024-11-26T20:43:41,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0c644311c9706b1426d8809c603e0fe8, UNASSIGN because future has completed 2024-11-26T20:43:41,524 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T20:43:41,524 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795}] 2024-11-26T20:43:41,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-26T20:43:41,676 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:41,676 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T20:43:41,676 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing 0c644311c9706b1426d8809c603e0fe8, disabling compactions & flushes 2024-11-26T20:43:41,676 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:41,676 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:41,676 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. after waiting 0 ms 2024-11-26T20:43:41,676 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:41,676 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing 0c644311c9706b1426d8809c603e0fe8 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-26T20:43:41,676 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=A 2024-11-26T20:43:41,676 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:41,676 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=B 2024-11-26T20:43:41,676 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:41,676 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0c644311c9706b1426d8809c603e0fe8, store=C 2024-11-26T20:43:41,676 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T20:43:41,681 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126dd7e6e5272804cf2bf1e371e1c19b26a_0c644311c9706b1426d8809c603e0fe8 is 50, key is test_row_0/A:col10/1732653819144/Put/seqid=0 2024-11-26T20:43:41,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742438_1614 (size=12454) 2024-11-26T20:43:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-26T20:43:42,085 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:43:42,088 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126dd7e6e5272804cf2bf1e371e1c19b26a_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126dd7e6e5272804cf2bf1e371e1c19b26a_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:42,088 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/fe265fb314844ddf8598f3d9091ed152, store: [table=TestAcidGuarantees family=A region=0c644311c9706b1426d8809c603e0fe8] 2024-11-26T20:43:42,089 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/fe265fb314844ddf8598f3d9091ed152 is 175, key is test_row_0/A:col10/1732653819144/Put/seqid=0 2024-11-26T20:43:42,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742439_1615 (size=31255) 2024-11-26T20:43:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-26T20:43:42,492 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/fe265fb314844ddf8598f3d9091ed152 2024-11-26T20:43:42,497 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/7f658516d9bc4969a646ce4499500beb is 50, key is test_row_0/B:col10/1732653819144/Put/seqid=0 2024-11-26T20:43:42,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742440_1616 (size=12301) 2024-11-26T20:43:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-26T20:43:42,900 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/7f658516d9bc4969a646ce4499500beb 2024-11-26T20:43:42,905 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/51347544bce24e189a261c2ef55b94b0 is 50, key is test_row_0/C:col10/1732653819144/Put/seqid=0 2024-11-26T20:43:42,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742441_1617 (size=12301) 2024-11-26T20:43:43,309 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/51347544bce24e189a261c2ef55b94b0 2024-11-26T20:43:43,312 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/A/fe265fb314844ddf8598f3d9091ed152 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/fe265fb314844ddf8598f3d9091ed152 2024-11-26T20:43:43,315 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/fe265fb314844ddf8598f3d9091ed152, entries=150, sequenceid=365, filesize=30.5 K 2024-11-26T20:43:43,316 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/B/7f658516d9bc4969a646ce4499500beb as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/7f658516d9bc4969a646ce4499500beb 2024-11-26T20:43:43,318 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/7f658516d9bc4969a646ce4499500beb, entries=150, sequenceid=365, filesize=12.0 K 2024-11-26T20:43:43,319 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/.tmp/C/51347544bce24e189a261c2ef55b94b0 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/51347544bce24e189a261c2ef55b94b0 2024-11-26T20:43:43,321 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/51347544bce24e189a261c2ef55b94b0, entries=150, sequenceid=365, filesize=12.0 K 2024-11-26T20:43:43,322 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0c644311c9706b1426d8809c603e0fe8 in 1646ms, sequenceid=365, compaction requested=true 2024-11-26T20:43:43,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/17aac43a6e224bd2b960c6b48edf9ff9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/bee9024d6d014948b6b583859ecca752, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/ee24d077689e475dbfc287b5dbf7f5de, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/c1222c836f3a4077a59f6617d41b8ad8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/b09dab8627d64a8d82d3144f208ff361, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/30c0567e74c9405199f5c695c8f73052, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/60c4b3bdd6a54e459c471bca76995f0d, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/09e85645a4814c8ea16c4bc194533159, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/25a45f9193d8481aa3ad2059b3c1cf6f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/6ab791b8bdf548b1be619762f3366e71, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/c65dad2614c942209f68fffbd637a63a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/eb0200d2fef847319c36ca453f19b6d1, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/1106645ffa8e4cccb35d376980e93c7a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/26ae3b70f2ab49c0ae9c04d179278741, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/df84956101094be3b344331dc83f3d12, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/161c4b216cf543f69e54017b9be8bb20, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/0b7039fcc27d4dee93f95143d3407067, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/260687fe68c445a095bfd9579f8d53c4, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/a34e952f70a9420faeed357a87eb4304, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/ee166d47f95c45bdad4abd8f738440b9, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/05c524de51e64d0db3e897f35d8a0bf1] to archive 2024-11-26T20:43:43,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T20:43:43,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/17aac43a6e224bd2b960c6b48edf9ff9 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/17aac43a6e224bd2b960c6b48edf9ff9 2024-11-26T20:43:43,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/bee9024d6d014948b6b583859ecca752 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/bee9024d6d014948b6b583859ecca752 2024-11-26T20:43:43,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/ee24d077689e475dbfc287b5dbf7f5de to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/ee24d077689e475dbfc287b5dbf7f5de 2024-11-26T20:43:43,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/c1222c836f3a4077a59f6617d41b8ad8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/c1222c836f3a4077a59f6617d41b8ad8 2024-11-26T20:43:43,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/b09dab8627d64a8d82d3144f208ff361 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/b09dab8627d64a8d82d3144f208ff361 2024-11-26T20:43:43,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/30c0567e74c9405199f5c695c8f73052 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/30c0567e74c9405199f5c695c8f73052 2024-11-26T20:43:43,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/60c4b3bdd6a54e459c471bca76995f0d to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/60c4b3bdd6a54e459c471bca76995f0d 2024-11-26T20:43:43,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/09e85645a4814c8ea16c4bc194533159 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/09e85645a4814c8ea16c4bc194533159 2024-11-26T20:43:43,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/25a45f9193d8481aa3ad2059b3c1cf6f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/25a45f9193d8481aa3ad2059b3c1cf6f 2024-11-26T20:43:43,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/6ab791b8bdf548b1be619762f3366e71 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/6ab791b8bdf548b1be619762f3366e71 2024-11-26T20:43:43,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/c65dad2614c942209f68fffbd637a63a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/c65dad2614c942209f68fffbd637a63a 2024-11-26T20:43:43,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/eb0200d2fef847319c36ca453f19b6d1 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/eb0200d2fef847319c36ca453f19b6d1 2024-11-26T20:43:43,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/1106645ffa8e4cccb35d376980e93c7a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/1106645ffa8e4cccb35d376980e93c7a 2024-11-26T20:43:43,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/26ae3b70f2ab49c0ae9c04d179278741 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/26ae3b70f2ab49c0ae9c04d179278741 2024-11-26T20:43:43,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/df84956101094be3b344331dc83f3d12 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/df84956101094be3b344331dc83f3d12 2024-11-26T20:43:43,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/161c4b216cf543f69e54017b9be8bb20 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/161c4b216cf543f69e54017b9be8bb20 2024-11-26T20:43:43,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/0b7039fcc27d4dee93f95143d3407067 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/0b7039fcc27d4dee93f95143d3407067 2024-11-26T20:43:43,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/260687fe68c445a095bfd9579f8d53c4 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/260687fe68c445a095bfd9579f8d53c4 2024-11-26T20:43:43,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/a34e952f70a9420faeed357a87eb4304 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/a34e952f70a9420faeed357a87eb4304 2024-11-26T20:43:43,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/ee166d47f95c45bdad4abd8f738440b9 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/ee166d47f95c45bdad4abd8f738440b9 2024-11-26T20:43:43,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/05c524de51e64d0db3e897f35d8a0bf1 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/05c524de51e64d0db3e897f35d8a0bf1 2024-11-26T20:43:43,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/bc1d0778308d49659bfc371e4a956c5e, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/ab6460139e8344dd8d0e43f0ef4f35b8, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/51328fe3d2d74e6583aa9b4f9f808eea, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/00d78b24c0d242ebaac2fe9ec66b4595, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/3048a20b0e674733b60ea4aedf7c2689, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/2ca4f86aacc24d63ba2bd0febfa3c47f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/0dd6916ccfdf4b1fb875966853fe7b93, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/9da92ffc94484f20be5b1aaf61edede3, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/f1f0abb83b4c4935b313f1db16201eb5, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/a5e32d22ef3047b2a3729e8949e5a93a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/4033bfbd06db484f8c6803b1bcf1bca4, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/96e70c9f070943f6a2eb9fa62b81d037, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/9ea2dc884f99456c996e798660427ece, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/b2e1122d37864eba97c71a676ac96636, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/6baa337f0db14c7bbdfaeed4d0f4b3f7, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/3bd89dfdca264a34982863a4bea649a4, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/a7282634c0a24671b8407e7c3efb9175, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/1b90b686febc47bbb49d0c3068a9e362, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/ccd7cefc98d244dcbfb65cfe6e75ed55, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/4067eb7e77b94e4087f9cc1599d2c7d0, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/3d1dcd729acf4c21b6687edbbd8584e8] to archive 2024-11-26T20:43:43,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T20:43:43,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/bc1d0778308d49659bfc371e4a956c5e to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/bc1d0778308d49659bfc371e4a956c5e 2024-11-26T20:43:43,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/ab6460139e8344dd8d0e43f0ef4f35b8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/ab6460139e8344dd8d0e43f0ef4f35b8 2024-11-26T20:43:43,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/51328fe3d2d74e6583aa9b4f9f808eea to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/51328fe3d2d74e6583aa9b4f9f808eea 2024-11-26T20:43:43,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/00d78b24c0d242ebaac2fe9ec66b4595 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/00d78b24c0d242ebaac2fe9ec66b4595 2024-11-26T20:43:43,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/3048a20b0e674733b60ea4aedf7c2689 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/3048a20b0e674733b60ea4aedf7c2689 2024-11-26T20:43:43,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/2ca4f86aacc24d63ba2bd0febfa3c47f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/2ca4f86aacc24d63ba2bd0febfa3c47f 2024-11-26T20:43:43,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/0dd6916ccfdf4b1fb875966853fe7b93 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/0dd6916ccfdf4b1fb875966853fe7b93 2024-11-26T20:43:43,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/9da92ffc94484f20be5b1aaf61edede3 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/9da92ffc94484f20be5b1aaf61edede3 2024-11-26T20:43:43,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/f1f0abb83b4c4935b313f1db16201eb5 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/f1f0abb83b4c4935b313f1db16201eb5 2024-11-26T20:43:43,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/a5e32d22ef3047b2a3729e8949e5a93a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/a5e32d22ef3047b2a3729e8949e5a93a 2024-11-26T20:43:43,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/4033bfbd06db484f8c6803b1bcf1bca4 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/4033bfbd06db484f8c6803b1bcf1bca4 2024-11-26T20:43:43,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/96e70c9f070943f6a2eb9fa62b81d037 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/96e70c9f070943f6a2eb9fa62b81d037 2024-11-26T20:43:43,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/9ea2dc884f99456c996e798660427ece to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/9ea2dc884f99456c996e798660427ece 2024-11-26T20:43:43,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/b2e1122d37864eba97c71a676ac96636 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/b2e1122d37864eba97c71a676ac96636 2024-11-26T20:43:43,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/6baa337f0db14c7bbdfaeed4d0f4b3f7 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/6baa337f0db14c7bbdfaeed4d0f4b3f7 2024-11-26T20:43:43,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/3bd89dfdca264a34982863a4bea649a4 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/3bd89dfdca264a34982863a4bea649a4 2024-11-26T20:43:43,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/a7282634c0a24671b8407e7c3efb9175 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/a7282634c0a24671b8407e7c3efb9175 2024-11-26T20:43:43,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/1b90b686febc47bbb49d0c3068a9e362 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/1b90b686febc47bbb49d0c3068a9e362 2024-11-26T20:43:43,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/ccd7cefc98d244dcbfb65cfe6e75ed55 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/ccd7cefc98d244dcbfb65cfe6e75ed55 2024-11-26T20:43:43,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/4067eb7e77b94e4087f9cc1599d2c7d0 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/4067eb7e77b94e4087f9cc1599d2c7d0 2024-11-26T20:43:43,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/3d1dcd729acf4c21b6687edbbd8584e8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/3d1dcd729acf4c21b6687edbbd8584e8 2024-11-26T20:43:43,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/ae1754899b0f41169026ae72cda10d84, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/fc0130d442304d87b84f419db3efa305, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/a396f3de1635436e99f7a51e06fe7b4f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/dd5dc25dec4e48929e0cee871cc5ab38, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/54fdfa821c4c43e28b7449f571748abf, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/10de564917394abfb679a9cba2349dcc, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/cdce5f863f7e40e1afa1987d32b22cda, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/6ea1109aec7448d19683fb9e11c9d181, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/ce260cc9f0fc44fba40b86e3d651e4bd, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/3acba7b2b44547af885a6c8a00d4516a, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/e0f48e35ee584629b891f59e0870238f, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/f526a745f73549eda4da59d03688fa98, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/528123e9fc804c82b16ca8c03809cb61, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/7bf4b5e011fc49fc80ce4acbf4316500, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/b8367dd515e34bea9404f0c0f1f0c9c7, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/3fe944a345e44de793f3cf119230d2c3, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/785876f9e97f42e4ba5cc3e353d25437, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/c4402ec8c6654bdaa48827bc1cc6a086, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/e8783c5674b341a09de0ed9bbe590b43, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/edb9dfa020df4321b0dde08d963de656, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/fd587b0ab51f447f93b5f3def5a51a66] to archive 2024-11-26T20:43:43,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T20:43:43,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/ae1754899b0f41169026ae72cda10d84 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/ae1754899b0f41169026ae72cda10d84 2024-11-26T20:43:43,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/fc0130d442304d87b84f419db3efa305 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/fc0130d442304d87b84f419db3efa305 2024-11-26T20:43:43,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/a396f3de1635436e99f7a51e06fe7b4f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/a396f3de1635436e99f7a51e06fe7b4f 2024-11-26T20:43:43,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/dd5dc25dec4e48929e0cee871cc5ab38 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/dd5dc25dec4e48929e0cee871cc5ab38 2024-11-26T20:43:43,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/54fdfa821c4c43e28b7449f571748abf to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/54fdfa821c4c43e28b7449f571748abf 2024-11-26T20:43:43,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/10de564917394abfb679a9cba2349dcc to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/10de564917394abfb679a9cba2349dcc 2024-11-26T20:43:43,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/cdce5f863f7e40e1afa1987d32b22cda to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/cdce5f863f7e40e1afa1987d32b22cda 2024-11-26T20:43:43,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/6ea1109aec7448d19683fb9e11c9d181 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/6ea1109aec7448d19683fb9e11c9d181 2024-11-26T20:43:43,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/ce260cc9f0fc44fba40b86e3d651e4bd to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/ce260cc9f0fc44fba40b86e3d651e4bd 2024-11-26T20:43:43,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/3acba7b2b44547af885a6c8a00d4516a to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/3acba7b2b44547af885a6c8a00d4516a 2024-11-26T20:43:43,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/e0f48e35ee584629b891f59e0870238f to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/e0f48e35ee584629b891f59e0870238f 2024-11-26T20:43:43,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/f526a745f73549eda4da59d03688fa98 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/f526a745f73549eda4da59d03688fa98 2024-11-26T20:43:43,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/528123e9fc804c82b16ca8c03809cb61 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/528123e9fc804c82b16ca8c03809cb61 2024-11-26T20:43:43,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/7bf4b5e011fc49fc80ce4acbf4316500 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/7bf4b5e011fc49fc80ce4acbf4316500 2024-11-26T20:43:43,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/b8367dd515e34bea9404f0c0f1f0c9c7 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/b8367dd515e34bea9404f0c0f1f0c9c7 2024-11-26T20:43:43,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/3fe944a345e44de793f3cf119230d2c3 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/3fe944a345e44de793f3cf119230d2c3 2024-11-26T20:43:43,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/785876f9e97f42e4ba5cc3e353d25437 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/785876f9e97f42e4ba5cc3e353d25437 2024-11-26T20:43:43,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/c4402ec8c6654bdaa48827bc1cc6a086 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/c4402ec8c6654bdaa48827bc1cc6a086 2024-11-26T20:43:43,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/e8783c5674b341a09de0ed9bbe590b43 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/e8783c5674b341a09de0ed9bbe590b43 2024-11-26T20:43:43,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/edb9dfa020df4321b0dde08d963de656 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/edb9dfa020df4321b0dde08d963de656 2024-11-26T20:43:43,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/fd587b0ab51f447f93b5f3def5a51a66 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/fd587b0ab51f447f93b5f3def5a51a66 2024-11-26T20:43:43,377 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/recovered.edits/368.seqid, newMaxSeqId=368, maxSeqId=4 2024-11-26T20:43:43,377 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. 2024-11-26T20:43:43,377 DEBUG [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for 0c644311c9706b1426d8809c603e0fe8: Waiting for close lock at 1732653821676Running coprocessor pre-close hooks at 1732653821676Disabling compacts and flushes for region at 1732653821676Disabling writes for close at 1732653821676Obtaining lock to block concurrent updates at 1732653821676Preparing flush snapshotting stores in 0c644311c9706b1426d8809c603e0fe8 at 1732653821676Finished memstore snapshotting TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732653821676Flushing stores of TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8. at 1732653821677 (+1 ms)Flushing 0c644311c9706b1426d8809c603e0fe8/A: creating writer at 1732653821677Flushing mob file 0c644311c9706b1426d8809c603e0fe8/A: appending metadata at 1732653821681 (+4 ms)Flushing mob file 0c644311c9706b1426d8809c603e0fe8/A: closing flushed file at 1732653821681Flushing 0c644311c9706b1426d8809c603e0fe8/A: appending metadata at 1732653822088 (+407 ms)Flushing 0c644311c9706b1426d8809c603e0fe8/A: closing flushed file at 1732653822088Flushing 0c644311c9706b1426d8809c603e0fe8/B: creating writer at 1732653822495 (+407 ms)Flushing 0c644311c9706b1426d8809c603e0fe8/B: appending metadata at 1732653822497 (+2 ms)Flushing 0c644311c9706b1426d8809c603e0fe8/B: closing flushed file at 1732653822497Flushing 0c644311c9706b1426d8809c603e0fe8/C: creating writer at 1732653822903 (+406 ms)Flushing 0c644311c9706b1426d8809c603e0fe8/C: appending metadata at 1732653822905 (+2 ms)Flushing 0c644311c9706b1426d8809c603e0fe8/C: closing flushed file at 1732653822905Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b3fb511: reopening flushed file at 1732653823312 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@340f3823: reopening flushed file at 1732653823315 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e46c356: reopening flushed file at 1732653823318 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0c644311c9706b1426d8809c603e0fe8 in 1646ms, sequenceid=365, compaction requested=true at 1732653823322 (+4 ms)Writing region close event to WAL at 1732653823374 (+52 ms)Running coprocessor post-close hooks at 1732653823377 (+3 ms)Closed at 1732653823377 2024-11-26T20:43:43,379 INFO [RS_CLOSE_REGION-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed 0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:43,379 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=0c644311c9706b1426d8809c603e0fe8, regionState=CLOSED 2024-11-26T20:43:43,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 because future has completed 2024-11-26T20:43:43,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-26T20:43:43,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure 0c644311c9706b1426d8809c603e0fe8, server=cd6be161543e,40291,1732653631795 in 1.8570 sec 2024-11-26T20:43:43,383 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-11-26T20:43:43,383 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0c644311c9706b1426d8809c603e0fe8, UNASSIGN in 1.8600 sec 2024-11-26T20:43:43,385 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-26T20:43:43,385 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8630 sec 2024-11-26T20:43:43,386 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732653823386"}]},"ts":"1732653823386"} 2024-11-26T20:43:43,387 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T20:43:43,387 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T20:43:43,388 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8700 sec 2024-11-26T20:43:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-26T20:43:43,664 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:43:43,664 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T20:43:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:43:43,666 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:43:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-26T20:43:43,667 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:43:43,669 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:43,670 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A, FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B, FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C, FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/recovered.edits] 2024-11-26T20:43:43,672 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/1dde5aa80874440f835efbb918375e86 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/1dde5aa80874440f835efbb918375e86 2024-11-26T20:43:43,673 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/b55107f6795b46cca3aed06b64f0b815 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/b55107f6795b46cca3aed06b64f0b815 2024-11-26T20:43:43,674 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/fe265fb314844ddf8598f3d9091ed152 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/A/fe265fb314844ddf8598f3d9091ed152 2024-11-26T20:43:43,676 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/1caf48edf2bc49629f7edb8f3b211b32 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/1caf48edf2bc49629f7edb8f3b211b32 2024-11-26T20:43:43,676 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/7f658516d9bc4969a646ce4499500beb to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/7f658516d9bc4969a646ce4499500beb 2024-11-26T20:43:43,677 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/985456ce31a641adb4e5d9d36ea2cd50 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/B/985456ce31a641adb4e5d9d36ea2cd50 2024-11-26T20:43:43,678 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/389c91cfefc94dc28ef7bcfce9908a79 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/389c91cfefc94dc28ef7bcfce9908a79 2024-11-26T20:43:43,679 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/51347544bce24e189a261c2ef55b94b0 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/51347544bce24e189a261c2ef55b94b0 2024-11-26T20:43:43,680 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/bef3498e4deb4986a81dfea48e06af01 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/C/bef3498e4deb4986a81dfea48e06af01 2024-11-26T20:43:43,682 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/recovered.edits/368.seqid to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8/recovered.edits/368.seqid 2024-11-26T20:43:43,683 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/default/TestAcidGuarantees/0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:43,683 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T20:43:43,683 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T20:43:43,683 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-26T20:43:43,686 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260ade8d35a7ee429ba5948b507c9eb30d_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260ade8d35a7ee429ba5948b507c9eb30d_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:43,686 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112611c93f90a3d24093975bdee8f0278902_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112611c93f90a3d24093975bdee8f0278902_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:43,687 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262c7d43b9e34140e99ca0c1fa4c3bd3d6_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262c7d43b9e34140e99ca0c1fa4c3bd3d6_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:43,688 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112652fb6dd90ccd4332a060868fa534ea02_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112652fb6dd90ccd4332a060868fa534ea02_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:43,689 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126714cf80d41874bf8bd42a1b3240cba06_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126714cf80d41874bf8bd42a1b3240cba06_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:43,690 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126834851ebf8be4a11b77bdf646df7fc6c_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126834851ebf8be4a11b77bdf646df7fc6c_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:43,690 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112692b8d340d1e84a859733acc893f3d403_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112692b8d340d1e84a859733acc893f3d403_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:43,691 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112694f5927233234d35afe1bf3e8fce3a10_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112694f5927233234d35afe1bf3e8fce3a10_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:43,692 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ac0fc2ffb61a4bcd9c807198dcb7a620_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ac0fc2ffb61a4bcd9c807198dcb7a620_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:43,693 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126aeff7e8911364f799c82faa4fe34293e_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126aeff7e8911364f799c82faa4fe34293e_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:43,694 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126bd2dce9d307f40c291d5fd3cf3486b6d_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126bd2dce9d307f40c291d5fd3cf3486b6d_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:43,694 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d2125163354943948db0c5294c1d39cb_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d2125163354943948db0c5294c1d39cb_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:43,695 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d6bb381e600f4386921d10bb7766de39_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d6bb381e600f4386921d10bb7766de39_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:43,696 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d76f2f278ee841efa96707a20f630ee2_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d76f2f278ee841efa96707a20f630ee2_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:43,697 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126dd7e6e5272804cf2bf1e371e1c19b26a_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126dd7e6e5272804cf2bf1e371e1c19b26a_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:43,698 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e4d479bf0e1f4e81948412fae76e3d8f_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e4d479bf0e1f4e81948412fae76e3d8f_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:43,698 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e85426934ed7452babd30a8e0c9ab5db_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e85426934ed7452babd30a8e0c9ab5db_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:43,699 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126eb2f28a0b633498383f42bc6b991fae6_0c644311c9706b1426d8809c603e0fe8 to hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126eb2f28a0b633498383f42bc6b991fae6_0c644311c9706b1426d8809c603e0fe8 2024-11-26T20:43:43,699 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T20:43:43,701 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:43:43,704 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T20:43:43,705 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T20:43:43,706 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:43:43,706 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T20:43:43,706 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732653823706"}]},"ts":"9223372036854775807"} 2024-11-26T20:43:43,707 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T20:43:43,707 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0c644311c9706b1426d8809c603e0fe8, NAME => 'TestAcidGuarantees,,1732653788512.0c644311c9706b1426d8809c603e0fe8.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T20:43:43,707 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T20:43:43,708 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732653823707"}]},"ts":"9223372036854775807"} 2024-11-26T20:43:43,709 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T20:43:43,709 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T20:43:43,710 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-26T20:43:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-26T20:43:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T20:43:43,774 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T20:43:43,783 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 248), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=366 (was 400), ProcessCount=11 (was 11), AvailableMemoryMB=5014 (was 5147) 2024-11-26T20:43:43,783 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T20:43:43,783 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T20:43:43,783 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:43:43,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:43,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:43,783 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:43:43,783 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T20:43:43,783 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1240002821, stopped=false 2024-11-26T20:43:43,784 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=cd6be161543e,42359,1732653630959 2024-11-26T20:43:43,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T20:43:43,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40291-0x10156022cfa0001, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T20:43:43,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:43:43,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40291-0x10156022cfa0001, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:43:43,786 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:43:43,786 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T20:43:43,786 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:43:43,786 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:43,787 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40291-0x10156022cfa0001, quorum=127.0.0.1:50761, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:43:43,787 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'cd6be161543e,40291,1732653631795' ***** 2024-11-26T20:43:43,787 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:43:43,787 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T20:43:43,787 INFO [RS:0;cd6be161543e:40291 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T20:43:43,788 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T20:43:43,788 INFO [RS:0;cd6be161543e:40291 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T20:43:43,788 INFO [RS:0;cd6be161543e:40291 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T20:43:43,788 INFO [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(959): stopping server cd6be161543e,40291,1732653631795 2024-11-26T20:43:43,788 INFO [RS:0;cd6be161543e:40291 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T20:43:43,788 INFO [RS:0;cd6be161543e:40291 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;cd6be161543e:40291. 2024-11-26T20:43:43,788 DEBUG [RS:0;cd6be161543e:40291 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:43:43,788 DEBUG [RS:0;cd6be161543e:40291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:43,788 INFO [RS:0;cd6be161543e:40291 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T20:43:43,788 INFO [RS:0;cd6be161543e:40291 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T20:43:43,788 INFO [RS:0;cd6be161543e:40291 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T20:43:43,788 INFO [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T20:43:43,789 INFO [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T20:43:43,789 DEBUG [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-26T20:43:43,789 DEBUG [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T20:43:43,789 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T20:43:43,789 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T20:43:43,789 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T20:43:43,789 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T20:43:43,789 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T20:43:43,789 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-26T20:43:43,813 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740/.tmp/info/fe90f502a7334d4b9f3ffd846f3ec69e is 95, key is TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3./info:/1732653665681/DeleteFamily/seqid=0 2024-11-26T20:43:43,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742442_1618 (size=6514) 2024-11-26T20:43:43,857 INFO [regionserver/cd6be161543e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T20:43:43,989 DEBUG [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T20:43:44,190 DEBUG [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T20:43:44,217 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740/.tmp/info/fe90f502a7334d4b9f3ffd846f3ec69e 2024-11-26T20:43:44,234 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740/.tmp/ns/1328ed3c19c24c9498cc4462e413dc93 is 93, key is TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3./ns:/1732653665632/DeleteFamily/seqid=0 2024-11-26T20:43:44,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742443_1619 (size=5906) 2024-11-26T20:43:44,390 DEBUG [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T20:43:44,590 DEBUG [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T20:43:44,637 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740/.tmp/ns/1328ed3c19c24c9498cc4462e413dc93 2024-11-26T20:43:44,654 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740/.tmp/rep_barrier/20ff6c26ed5942e9a00bdd4ca7dac263 is 102, key is TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3./rep_barrier:/1732653665632/DeleteFamily/seqid=0 2024-11-26T20:43:44,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742444_1620 (size=6025) 2024-11-26T20:43:44,768 INFO [regionserver/cd6be161543e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-26T20:43:44,768 INFO [regionserver/cd6be161543e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-26T20:43:44,790 DEBUG [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T20:43:44,990 DEBUG [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T20:43:45,058 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740/.tmp/rep_barrier/20ff6c26ed5942e9a00bdd4ca7dac263 2024-11-26T20:43:45,075 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740/.tmp/table/5325b3494d454d31b768bbc2b2e0b66f is 96, key is TestAcidGuarantees,,1732653635348.f47d720597d62d3cb60d84298295c6f3./table:/1732653665632/DeleteFamily/seqid=0 2024-11-26T20:43:45,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742445_1621 (size=5936) 2024-11-26T20:43:45,191 DEBUG [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T20:43:45,391 DEBUG [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T20:43:45,478 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740/.tmp/table/5325b3494d454d31b768bbc2b2e0b66f 2024-11-26T20:43:45,482 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740/.tmp/info/fe90f502a7334d4b9f3ffd846f3ec69e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740/info/fe90f502a7334d4b9f3ffd846f3ec69e 2024-11-26T20:43:45,484 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740/info/fe90f502a7334d4b9f3ffd846f3ec69e, entries=12, sequenceid=90, filesize=6.4 K 2024-11-26T20:43:45,485 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740/.tmp/ns/1328ed3c19c24c9498cc4462e413dc93 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740/ns/1328ed3c19c24c9498cc4462e413dc93 2024-11-26T20:43:45,487 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740/ns/1328ed3c19c24c9498cc4462e413dc93, entries=8, sequenceid=90, filesize=5.8 K 2024-11-26T20:43:45,488 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740/.tmp/rep_barrier/20ff6c26ed5942e9a00bdd4ca7dac263 as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740/rep_barrier/20ff6c26ed5942e9a00bdd4ca7dac263 2024-11-26T20:43:45,491 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740/rep_barrier/20ff6c26ed5942e9a00bdd4ca7dac263, entries=6, sequenceid=90, filesize=5.9 K 2024-11-26T20:43:45,491 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740/.tmp/table/5325b3494d454d31b768bbc2b2e0b66f as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740/table/5325b3494d454d31b768bbc2b2e0b66f 2024-11-26T20:43:45,494 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740/table/5325b3494d454d31b768bbc2b2e0b66f, entries=7, sequenceid=90, filesize=5.8 K 2024-11-26T20:43:45,494 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1705ms, sequenceid=90, compaction requested=false 2024-11-26T20:43:45,498 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-26T20:43:45,498 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T20:43:45,499 INFO [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T20:43:45,499 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732653823789Running coprocessor pre-close hooks at 1732653823789Disabling compacts and flushes for region at 1732653823789Disabling writes for close at 1732653823789Obtaining lock to block concurrent updates at 1732653823789Preparing flush snapshotting stores in 1588230740 at 1732653823789Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732653823790 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732653823791 (+1 ms)Flushing 1588230740/info: creating writer at 1732653823791Flushing 1588230740/info: appending metadata at 1732653823813 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732653823813Flushing 1588230740/ns: creating writer at 1732653824220 (+407 ms)Flushing 1588230740/ns: appending metadata at 1732653824233 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1732653824233Flushing 1588230740/rep_barrier: creating writer at 1732653824640 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732653824654 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732653824654Flushing 1588230740/table: creating writer at 1732653825061 (+407 ms)Flushing 1588230740/table: appending metadata at 1732653825075 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732653825075Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@100a555f: reopening flushed file at 1732653825481 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f075f3a: reopening flushed file at 1732653825485 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ba77702: reopening flushed file at 1732653825488 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c81a21: reopening flushed file at 1732653825491 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1705ms, sequenceid=90, compaction requested=false at 1732653825494 (+3 ms)Writing region close event to WAL at 1732653825496 (+2 ms)Running coprocessor post-close hooks at 1732653825498 (+2 ms)Closed at 1732653825498 2024-11-26T20:43:45,499 DEBUG [RS_CLOSE_META-regionserver/cd6be161543e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T20:43:45,591 INFO [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(976): stopping server cd6be161543e,40291,1732653631795; all regions closed. 2024-11-26T20:43:45,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741834_1010 (size=25344) 2024-11-26T20:43:45,597 DEBUG [RS:0;cd6be161543e:40291 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/oldWALs 2024-11-26T20:43:45,597 INFO [RS:0;cd6be161543e:40291 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL cd6be161543e%2C40291%2C1732653631795.meta:.meta(num 1732653634887) 2024-11-26T20:43:45,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741832_1008 (size=16672071) 2024-11-26T20:43:45,600 DEBUG [RS:0;cd6be161543e:40291 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/oldWALs 2024-11-26T20:43:45,600 INFO [RS:0;cd6be161543e:40291 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL cd6be161543e%2C40291%2C1732653631795:(num 1732653633935) 2024-11-26T20:43:45,600 DEBUG [RS:0;cd6be161543e:40291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:43:45,601 INFO [RS:0;cd6be161543e:40291 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T20:43:45,601 INFO [RS:0;cd6be161543e:40291 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T20:43:45,601 INFO [RS:0;cd6be161543e:40291 {}] hbase.ChoreService(370): Chore service for: regionserver/cd6be161543e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T20:43:45,601 INFO [RS:0;cd6be161543e:40291 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T20:43:45,601 INFO [regionserver/cd6be161543e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T20:43:45,601 INFO [RS:0;cd6be161543e:40291 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40291 2024-11-26T20:43:45,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40291-0x10156022cfa0001, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/cd6be161543e,40291,1732653631795 2024-11-26T20:43:45,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T20:43:45,606 INFO [RS:0;cd6be161543e:40291 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T20:43:45,609 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [cd6be161543e,40291,1732653631795] 2024-11-26T20:43:45,610 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/cd6be161543e,40291,1732653631795 already deleted, retry=false 2024-11-26T20:43:45,610 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; cd6be161543e,40291,1732653631795 expired; onlineServers=0 2024-11-26T20:43:45,610 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'cd6be161543e,42359,1732653630959' ***** 2024-11-26T20:43:45,610 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T20:43:45,610 INFO [M:0;cd6be161543e:42359 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T20:43:45,611 INFO [M:0;cd6be161543e:42359 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T20:43:45,611 DEBUG [M:0;cd6be161543e:42359 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T20:43:45,611 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T20:43:45,611 DEBUG [M:0;cd6be161543e:42359 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T20:43:45,611 DEBUG [master/cd6be161543e:0:becomeActiveMaster-HFileCleaner.large.0-1732653633650 {}] cleaner.HFileCleaner(306): Exit Thread[master/cd6be161543e:0:becomeActiveMaster-HFileCleaner.large.0-1732653633650,5,FailOnTimeoutGroup] 2024-11-26T20:43:45,611 DEBUG [master/cd6be161543e:0:becomeActiveMaster-HFileCleaner.small.0-1732653633657 {}] cleaner.HFileCleaner(306): Exit Thread[master/cd6be161543e:0:becomeActiveMaster-HFileCleaner.small.0-1732653633657,5,FailOnTimeoutGroup] 2024-11-26T20:43:45,611 INFO [M:0;cd6be161543e:42359 {}] hbase.ChoreService(370): Chore service for: master/cd6be161543e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T20:43:45,611 INFO [M:0;cd6be161543e:42359 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T20:43:45,611 DEBUG [M:0;cd6be161543e:42359 {}] master.HMaster(1795): Stopping service threads 2024-11-26T20:43:45,611 INFO [M:0;cd6be161543e:42359 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T20:43:45,611 INFO [M:0;cd6be161543e:42359 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:43:45,611 ERROR [M:0;cd6be161543e:42359 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1228365231) connection to localhost/127.0.0.1:39285 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:39285,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-26T20:43:45,612 INFO [M:0;cd6be161543e:42359 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:43:45,612 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T20:43:45,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T20:43:45,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:43:45,613 DEBUG [M:0;cd6be161543e:42359 {}] zookeeper.ZKUtil(347): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T20:43:45,613 WARN [M:0;cd6be161543e:42359 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T20:43:45,613 INFO [M:0;cd6be161543e:42359 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/.lastflushedseqids 2024-11-26T20:43:45,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742446_1622 (size=72) 2024-11-26T20:43:45,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40291-0x10156022cfa0001, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:43:45,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40291-0x10156022cfa0001, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:43:45,710 INFO [RS:0;cd6be161543e:40291 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T20:43:45,710 INFO [RS:0;cd6be161543e:40291 {}] regionserver.HRegionServer(1031): Exiting; stopping=cd6be161543e,40291,1732653631795; zookeeper connection closed. 2024-11-26T20:43:45,710 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@a44e11 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@a44e11 2024-11-26T20:43:45,710 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-26T20:43:46,023 INFO [M:0;cd6be161543e:42359 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:43:46,023 INFO [M:0;cd6be161543e:42359 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T20:43:46,024 DEBUG [M:0;cd6be161543e:42359 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T20:43:46,024 INFO [M:0;cd6be161543e:42359 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:43:46,024 DEBUG [M:0;cd6be161543e:42359 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:43:46,024 DEBUG [M:0;cd6be161543e:42359 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T20:43:46,024 DEBUG [M:0;cd6be161543e:42359 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:43:46,024 INFO [M:0;cd6be161543e:42359 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=755.11 KB heapSize=923.59 KB 2024-11-26T20:43:46,039 DEBUG [M:0;cd6be161543e:42359 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/babfe61aa3df49fca4ed939da84c163e is 82, key is hbase:meta,,1/info:regioninfo/1732653634969/Put/seqid=0 2024-11-26T20:43:46,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742447_1623 (size=5672) 2024-11-26T20:43:46,442 INFO [M:0;cd6be161543e:42359 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2075 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/babfe61aa3df49fca4ed939da84c163e 2024-11-26T20:43:46,461 DEBUG [M:0;cd6be161543e:42359 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/648df8251a1445889086156d98de97ce is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x8D/proc:d/1732653791586/Put/seqid=0 2024-11-26T20:43:46,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742448_1624 (size=45002) 2024-11-26T20:43:46,865 INFO [M:0;cd6be161543e:42359 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=754.55 KB at sequenceid=2075 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/648df8251a1445889086156d98de97ce 2024-11-26T20:43:46,868 INFO [M:0;cd6be161543e:42359 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 648df8251a1445889086156d98de97ce 2024-11-26T20:43:46,883 DEBUG [M:0;cd6be161543e:42359 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/446a6c12286b4919bf2622fe75c0883b is 69, key is cd6be161543e,40291,1732653631795/rs:state/1732653633671/Put/seqid=0 2024-11-26T20:43:46,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073742449_1625 (size=5156) 2024-11-26T20:43:47,286 INFO [M:0;cd6be161543e:42359 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2075 (bloomFilter=true), to=hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/446a6c12286b4919bf2622fe75c0883b 2024-11-26T20:43:47,289 DEBUG [M:0;cd6be161543e:42359 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/babfe61aa3df49fca4ed939da84c163e as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/babfe61aa3df49fca4ed939da84c163e 2024-11-26T20:43:47,292 INFO [M:0;cd6be161543e:42359 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/babfe61aa3df49fca4ed939da84c163e, entries=8, sequenceid=2075, filesize=5.5 K 2024-11-26T20:43:47,292 DEBUG [M:0;cd6be161543e:42359 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/648df8251a1445889086156d98de97ce as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/648df8251a1445889086156d98de97ce 2024-11-26T20:43:47,295 INFO [M:0;cd6be161543e:42359 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 648df8251a1445889086156d98de97ce 2024-11-26T20:43:47,295 INFO [M:0;cd6be161543e:42359 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/648df8251a1445889086156d98de97ce, entries=168, sequenceid=2075, filesize=43.9 K 2024-11-26T20:43:47,295 DEBUG [M:0;cd6be161543e:42359 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/446a6c12286b4919bf2622fe75c0883b as hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/446a6c12286b4919bf2622fe75c0883b 2024-11-26T20:43:47,297 INFO [M:0;cd6be161543e:42359 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39285/user/jenkins/test-data/07bafd68-651d-d3fc-39e3-42e1c980f794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/446a6c12286b4919bf2622fe75c0883b, entries=1, sequenceid=2075, filesize=5.0 K 2024-11-26T20:43:47,298 INFO [M:0;cd6be161543e:42359 {}] regionserver.HRegion(3140): Finished flush of dataSize ~755.11 KB/773232, heapSize ~923.29 KB/945448, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1274ms, sequenceid=2075, compaction requested=false 2024-11-26T20:43:47,299 INFO [M:0;cd6be161543e:42359 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:43:47,299 DEBUG [M:0;cd6be161543e:42359 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732653826024Disabling compacts and flushes for region at 1732653826024Disabling writes for close at 1732653826024Obtaining lock to block concurrent updates at 1732653826024Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732653826024Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=773232, getHeapSize=945688, getOffHeapSize=0, getCellsCount=2268 at 1732653826024Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732653826025 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732653826025Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732653826038 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732653826038Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732653826445 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732653826461 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732653826461Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732653826869 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732653826882 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732653826882Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3aa1ccf8: reopening flushed file at 1732653827289 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16509244: reopening flushed file at 1732653827292 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c708e07: reopening flushed file at 1732653827295 (+3 ms)Finished flush of dataSize ~755.11 KB/773232, heapSize ~923.29 KB/945448, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1274ms, sequenceid=2075, compaction requested=false at 1732653827298 (+3 ms)Writing region close event to WAL at 1732653827299 (+1 ms)Closed at 1732653827299 2024-11-26T20:43:47,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34989 is added to blk_1073741830_1006 (size=909298) 2024-11-26T20:43:47,302 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T20:43:47,302 INFO [M:0;cd6be161543e:42359 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T20:43:47,302 INFO [M:0;cd6be161543e:42359 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42359 2024-11-26T20:43:47,302 INFO [M:0;cd6be161543e:42359 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T20:43:47,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:43:47,406 INFO [M:0;cd6be161543e:42359 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T20:43:47,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x10156022cfa0000, quorum=127.0.0.1:50761, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:43:47,410 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T20:43:47,412 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T20:43:47,412 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T20:43:47,412 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T20:43:47,413 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/hadoop.log.dir/,STOPPED} 2024-11-26T20:43:47,415 WARN [BP-376524052-172.17.0.2-1732653627608 heartbeating to localhost/127.0.0.1:39285 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T20:43:47,415 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T20:43:47,415 WARN [BP-376524052-172.17.0.2-1732653627608 heartbeating to localhost/127.0.0.1:39285 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-376524052-172.17.0.2-1732653627608 (Datanode Uuid 6f31a597-6d37-4855-9b75-9219c1156dc9) service to localhost/127.0.0.1:39285 2024-11-26T20:43:47,415 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T20:43:47,417 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/cluster_bc121de7-746a-3d7e-5c7c-57aa2e103a17/data/data1/current/BP-376524052-172.17.0.2-1732653627608 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T20:43:47,418 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/cluster_bc121de7-746a-3d7e-5c7c-57aa2e103a17/data/data2/current/BP-376524052-172.17.0.2-1732653627608 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T20:43:47,418 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T20:43:47,425 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T20:43:47,426 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T20:43:47,426 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T20:43:47,426 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T20:43:47,426 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9e2f0d6-db0d-d6c2-aad1-30deeb529e49/hadoop.log.dir/,STOPPED} 2024-11-26T20:43:47,447 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T20:43:47,562 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down